You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2014/04/09 19:57:32 UTC
[01/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Repository: accumulo
Updated Branches:
refs/heads/1.4.6-SNAPSHOT 0f6392400 -> c8e165a31
refs/heads/1.5.2-SNAPSHOT 5363d781d -> 926133889
refs/heads/1.6.0-SNAPSHOT 3934ea6a2 -> 716ea0ee8
refs/heads/master 8856c1f66 -> 46b0f986d
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
index 06b9a7c,0000000..70156b2
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
@@@ -1,223 -1,0 +1,222 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.continuous;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.ClientOnDefaultTable;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.server.util.reflection.CounterUtils;
+import org.apache.accumulo.test.continuous.ContinuousWalk.BadChecksumException;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.VLongWritable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.validators.PositiveInteger;
+
+/**
+ * A map reduce job that verifies a table created by continuous ingest. It verifies that all referenced nodes are defined.
+ */
+
+public class ContinuousVerify extends Configured implements Tool {
+ public static final VLongWritable DEF = new VLongWritable(-1);
+
+ public static class CMapper extends Mapper<Key,Value,LongWritable,VLongWritable> {
+
+ private LongWritable row = new LongWritable();
+ private LongWritable ref = new LongWritable();
+ private VLongWritable vrow = new VLongWritable();
+
+ private long corrupt = 0;
+
+ @Override
+ public void map(Key key, Value data, Context context) throws IOException, InterruptedException {
+ long r = Long.parseLong(key.getRow().toString(), 16);
+ if (r < 0)
+ throw new IllegalArgumentException();
+
+ try {
+ ContinuousWalk.validate(key, data);
+ } catch (BadChecksumException bce) {
+ CounterUtils.increment(context.getCounter(Counts.CORRUPT));
+ if (corrupt < 1000) {
+ System.out.println("ERROR Bad checksum : " + key);
+ } else if (corrupt == 1000) {
+ System.out.println("Too many bad checksums, not printing anymore!");
+ }
+ corrupt++;
+ return;
+ }
+
+ row.set(r);
+
+ context.write(row, DEF);
+ byte[] val = data.get();
+
+ int offset = ContinuousWalk.getPrevRowOffset(val);
+ if (offset > 0) {
+ ref.set(Long.parseLong(new String(val, offset, 16, Constants.UTF8), 16));
+ vrow.set(r);
+ context.write(ref, vrow);
+ }
+ }
+ }
+
+ public static enum Counts {
+ UNREFERENCED, UNDEFINED, REFERENCED, CORRUPT
+ }
+
+ public static class CReducer extends Reducer<LongWritable,VLongWritable,Text,Text> {
+ private ArrayList<Long> refs = new ArrayList<Long>();
+
+ @Override
+ public void reduce(LongWritable key, Iterable<VLongWritable> values, Context context) throws IOException, InterruptedException {
+
+ int defCount = 0;
+
+ refs.clear();
+ for (VLongWritable type : values) {
+ if (type.get() == -1) {
+ defCount++;
+ } else {
+ refs.add(type.get());
+ }
+ }
+
+ if (defCount == 0 && refs.size() > 0) {
+ StringBuilder sb = new StringBuilder();
+ String comma = "";
+ for (Long ref : refs) {
+ sb.append(comma);
+ comma = ",";
+ sb.append(new String(ContinuousIngest.genRow(ref), Constants.UTF8));
+ }
+
+ context.write(new Text(ContinuousIngest.genRow(key.get())), new Text(sb.toString()));
+ CounterUtils.increment(context.getCounter(Counts.UNDEFINED));
+
+ } else if (defCount > 0 && refs.size() == 0) {
+ CounterUtils.increment(context.getCounter(Counts.UNREFERENCED));
+ } else {
+ CounterUtils.increment(context.getCounter(Counts.REFERENCED));
+ }
+
+ }
+ }
+
+ static class Opts extends ClientOnDefaultTable {
+ @Parameter(names = "--output", description = "location in HDFS to store the results; must not exist", required = true)
+ String outputDir = "/tmp/continuousVerify";
+
+ @Parameter(names = "--maxMappers", description = "the maximum number of mappers to use", required = true, validateWith = PositiveInteger.class)
+ int maxMaps = 0;
+
+ @Parameter(names = "--reducers", description = "the number of reducers to use", required = true, validateWith = PositiveInteger.class)
+ int reducers = 0;
+
+ @Parameter(names = "--offline", description = "perform the verification directly on the files while the table is offline")
+ boolean scanOffline = false;
+
+ public Opts() {
+ super("ci");
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(this.getClass().getName(), args);
+
+ Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+ job.setJarByClass(this.getClass());
+
+ job.setInputFormatClass(AccumuloInputFormat.class);
+ opts.setAccumuloConfigs(job);
+
+ Set<Range> ranges = null;
+ String clone = opts.getTableName();
+ Connector conn = null;
+
+ if (opts.scanOffline) {
+ Random random = new Random();
+ clone = opts.getTableName() + "_" + String.format("%016x", (random.nextLong() & 0x7fffffffffffffffl));
+ conn = opts.getConnector();
+ conn.tableOperations().clone(opts.getTableName(), clone, true, new HashMap<String,String>(), new HashSet<String>());
+ ranges = conn.tableOperations().splitRangeByTablets(opts.getTableName(), new Range(), opts.maxMaps);
+ conn.tableOperations().offline(clone);
+ AccumuloInputFormat.setInputTableName(job, clone);
+ AccumuloInputFormat.setOfflineTableScan(job, true);
+ } else {
+ ranges = opts.getConnector().tableOperations().splitRangeByTablets(opts.getTableName(), new Range(), opts.maxMaps);
+ }
+
+ AccumuloInputFormat.setRanges(job, ranges);
+ AccumuloInputFormat.setAutoAdjustRanges(job, false);
+
+ job.setMapperClass(CMapper.class);
+ job.setMapOutputKeyClass(LongWritable.class);
+ job.setMapOutputValueClass(VLongWritable.class);
+
+ job.setReducerClass(CReducer.class);
+ job.setNumReduceTasks(opts.reducers);
+
+ job.setOutputFormatClass(TextOutputFormat.class);
+
+ job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", opts.scanOffline);
+
+ TextOutputFormat.setOutputPath(job, new Path(opts.outputDir));
+
+ job.waitForCompletion(true);
+
+ if (opts.scanOffline) {
+ conn.tableOperations().delete(clone);
+ }
+ opts.stopTracing();
+ return job.isSuccessful() ? 0 : 1;
+ }
+
+ /**
+ *
+ * @param args
+ * instanceName zookeepers username password table columns outputpath
- * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(CachedConfiguration.getInstance(), new ContinuousVerify(), args);
+ if (res != 0)
+ System.exit(res);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/functional/CacheTestClean.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/CacheTestClean.java
index c522914,0000000..f1dfcd2
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/CacheTestClean.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/CacheTestClean.java
@@@ -1,51 -1,0 +1,48 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import java.io.File;
+import java.util.Arrays;
+
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+
+public class CacheTestClean {
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ String rootDir = args[0];
+ File reportDir = new File(args[1]);
+
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ if (zoo.exists(rootDir)) {
+ zoo.recursiveDelete(rootDir, NodeMissingPolicy.FAIL);
+ }
+
+ if (!reportDir.exists()) {
+ reportDir.mkdir();
+ } else {
+ File[] files = reportDir.listFiles();
+ if (files.length != 0)
+ throw new Exception("dir " + reportDir + " is not empty: " + Arrays.asList(files));
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
index 2b775c5,0000000..06c6fdb
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
@@@ -1,217 -1,0 +1,213 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.server.util.reflection.CounterUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Runs the functional tests via map-reduce.
+ *
+ * First, be sure everything is compiled.
+ *
+ * Second, get a list of the tests you want to run:
+ *
+ * <pre>
+ * $ python test/system/auto/run.py -l > tests
+ * </pre>
+ *
+ * Put the list of tests into HDFS:
+ *
+ * <pre>
+ * $ hadoop fs -put tests /user/hadoop/tests
+ * </pre>
+ *
+ * Run the map-reduce job:
+ *
+ * <pre>
+ * $ ./bin/accumulo accumulo.test.functional.RunTests --tests /user/hadoop/tests --output /user/hadoop/results
+ * </pre>
+ *
+ * Note that you will need to have some configuration in conf/accumulo-site.xml (to locate zookeeper). The map-reduce jobs will not use your local accumulo
+ * instance.
+ *
+ */
+public class RunTests extends Configured implements Tool {
+
+ static final public String JOB_NAME = "Functional Test Runner";
+ private static final Logger log = Logger.getLogger(RunTests.class);
+
+ private Job job = null;
+
+ private static final int DEFAULT_TIMEOUT_FACTOR = 1;
+
+ static class Opts extends Help {
+ @Parameter(names="--tests", description="newline separated list of tests to run", required=true)
+ String testFile;
+ @Parameter(names="--output", description="destination for the results of tests in HDFS", required=true)
+ String outputPath;
+ @Parameter(names="--timeoutFactor", description="Optional scaling factor for timeout for both mapred.task.timeout and -f flag on run.py", required=false)
+ Integer intTimeoutFactor = DEFAULT_TIMEOUT_FACTOR;
+ }
+
+ static final String TIMEOUT_FACTOR = RunTests.class.getName() + ".timeoutFactor";
+
+ static public class TestMapper extends Mapper<LongWritable,Text,Text,Text> {
+
+ private static final String REDUCER_RESULT_START = "::::: ";
+ private static final int RRS_LEN = REDUCER_RESULT_START.length();
+ private Text result = new Text();
+ String mapperTimeoutFactor = null;
+
+ private static enum Outcome {
+ SUCCESS, FAILURE, ERROR, UNEXPECTED_SUCCESS, EXPECTED_FAILURE
+ }
+ private static final Map<Character, Outcome> OUTCOME_COUNTERS;
+ static {
+ OUTCOME_COUNTERS = new java.util.HashMap<Character, Outcome>();
+ OUTCOME_COUNTERS.put('S', Outcome.SUCCESS);
+ OUTCOME_COUNTERS.put('F', Outcome.FAILURE);
+ OUTCOME_COUNTERS.put('E', Outcome.ERROR);
+ OUTCOME_COUNTERS.put('T', Outcome.UNEXPECTED_SUCCESS);
+ OUTCOME_COUNTERS.put('G', Outcome.EXPECTED_FAILURE);
+ }
+
+ @Override
+ protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
+ List<String> cmd = Arrays.asList("/usr/bin/python", "test/system/auto/run.py", "-m", "-f", mapperTimeoutFactor, "-t", value.toString());
+ log.info("Running test " + cmd);
+ ProcessBuilder pb = new ProcessBuilder(cmd);
+ pb.directory(new File(context.getConfiguration().get("accumulo.home")));
+ pb.redirectErrorStream(true);
+ Process p = pb.start();
+ p.getOutputStream().close();
+ InputStream out = p.getInputStream();
+ InputStreamReader outr = new InputStreamReader(out, Constants.UTF8);
+ BufferedReader br = new BufferedReader(outr);
+ String line;
+ try {
+ while ((line = br.readLine()) != null) {
+ log.info("More: " + line);
+ if (line.startsWith(REDUCER_RESULT_START)) {
+ String resultLine = line.substring(RRS_LEN);
+ if (resultLine.length() > 0) {
+ Outcome outcome = OUTCOME_COUNTERS.get(resultLine.charAt(0));
+ if (outcome != null) {
+ CounterUtils.increment(context.getCounter(outcome));
+ }
+ }
+ String taskAttemptId = context.getTaskAttemptID().toString();
+ result.set(taskAttemptId + " " + resultLine);
+ context.write(value, result);
+ }
+ }
+ } catch (Exception ex) {
+ log.error(ex);
+ context.progress();
+ }
+
+ p.waitFor();
+ }
+
+ @Override
+ protected void setup(Mapper<LongWritable,Text,Text,Text>.Context context) throws IOException, InterruptedException {
+ mapperTimeoutFactor = Integer.toString(context.getConfiguration().getInt(TIMEOUT_FACTOR, DEFAULT_TIMEOUT_FACTOR));
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ job = new Job(getConf(), JOB_NAME);
+ job.setJarByClass(this.getClass());
+ Opts opts = new Opts();
+ opts.parseArgs(RunTests.class.getName(), args);
+
+ // this is like 1-2 tests per mapper
+ Configuration conf = job.getConfiguration();
+ conf.setInt("mapred.max.split.size", 40);
+ conf.set("accumulo.home", System.getenv("ACCUMULO_HOME"));
+
+ // Taking third argument as scaling factor to setting mapred.task.timeout
+ // and TIMEOUT_FACTOR
+ conf.setInt("mapred.task.timeout", opts.intTimeoutFactor * 8 * 60 * 1000);
+ conf.setInt(TIMEOUT_FACTOR, opts.intTimeoutFactor);
+ conf.setBoolean("mapred.map.tasks.speculative.execution", false);
+
+ // set input
+ job.setInputFormatClass(TextInputFormat.class);
+ TextInputFormat.setInputPaths(job, new Path(opts.testFile));
+
+ // set output
+ job.setOutputFormatClass(TextOutputFormat.class);
+ FileSystem fs = FileSystem.get(conf);
+ Path destination = new Path(opts.outputPath);
+ if (fs.exists(destination)) {
+ log.info("Deleting existing output directory " + opts.outputPath);
+ fs.delete(destination, true);
+ }
+ TextOutputFormat.setOutputPath(job, destination);
+
+ // configure default reducer: put the results into one file
+ job.setNumReduceTasks(1);
+
+ // set mapper
+ job.setMapperClass(TestMapper.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(Text.class);
+
+ // don't do anything with the results (yet) a summary would be nice
+ job.setNumReduceTasks(0);
+
+ // submit the job
+ log.info("Starting tests");
+ return 0;
+ }
+
- /**
- * @param args
- * @throws Exception
- */
+ public static void main(String[] args) throws Exception {
+ RunTests tests = new RunTests();
+ ToolRunner.run(new Configuration(), tests, args);
+ tests.job.waitForCompletion(true);
+ if (!tests.job.isSuccessful())
+ System.exit(1);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
index a9b072e,0000000..85cddbb
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
@@@ -1,246 -1,0 +1,243 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.performance.metadata;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.TreeSet;
+import java.util.UUID;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.util.Stat;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.hadoop.io.Text;
+
+/**
+ * This little program can be used to write a lot of entries to the !METADATA table and measure the performance of varying numbers of threads doing !METADATA
+ * lookups using the batch scanner.
+ *
+ *
+ */
+
+public class MetadataBatchScanTest {
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+
+ final Connector connector = new ZooKeeperInstance("acu14", "localhost")
+ .getConnector(SecurityConstants.SYSTEM_PRINCIPAL, SecurityConstants.getSystemToken());
+
+ TreeSet<Long> splits = new TreeSet<Long>();
+ Random r = new Random(42);
+
+ while (splits.size() < 99999) {
+ splits.add((r.nextLong() & 0x7fffffffffffffffl) % 1000000000000l);
+ }
+
+ Text tid = new Text("8");
+ Text per = null;
+
+ ArrayList<KeyExtent> extents = new ArrayList<KeyExtent>();
+
+ for (Long split : splits) {
+ Text er = new Text(String.format("%012d", split));
+ KeyExtent ke = new KeyExtent(tid, er, per);
+ per = er;
+
+ extents.add(ke);
+ }
+
+ extents.add(new KeyExtent(tid, null, per));
+
+ if (args[0].equals("write")) {
+
+ BatchWriter bw = connector.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig());
+
+ for (KeyExtent extent : extents) {
+ Mutation mut = extent.getPrevRowUpdateMutation();
+ new TServerInstance(AddressUtil.parseAddress("192.168.1.100", 4567), "DEADBEEF").putLocation(mut);
+ bw.addMutation(mut);
+ }
+
+ bw.close();
+ } else if (args[0].equals("writeFiles")) {
+ BatchWriter bw = connector.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig());
+
+ for (KeyExtent extent : extents) {
+
+ Mutation mut = new Mutation(extent.getMetadataEntry());
+
+ String dir = "/t-" + UUID.randomUUID();
+
+ Constants.METADATA_DIRECTORY_COLUMN.put(mut, new Value(dir.getBytes(Constants.UTF8)));
+
+ for (int i = 0; i < 5; i++) {
+ mut.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(dir + "/00000_0000" + i + ".map"), new Value("10000,1000000".getBytes(Constants.UTF8)));
+ }
+
+ bw.addMutation(mut);
+ }
+
+ bw.close();
+ } else if (args[0].equals("scan")) {
+
+ int numThreads = Integer.parseInt(args[1]);
+ final int numLoop = Integer.parseInt(args[2]);
+ int numLookups = Integer.parseInt(args[3]);
+
+ HashSet<Integer> indexes = new HashSet<Integer>();
+ while (indexes.size() < numLookups) {
+ indexes.add(r.nextInt(extents.size()));
+ }
+
+ final List<Range> ranges = new ArrayList<Range>();
+ for (Integer i : indexes) {
+ ranges.add(extents.get(i).toMetadataRange());
+ }
+
+ Thread threads[] = new Thread[numThreads];
+
+ for (int i = 0; i < threads.length; i++) {
+ threads[i] = new Thread(new Runnable() {
+
+ @Override
+ public void run() {
+ try {
+ System.out.println(runScanTest(connector, numLoop, ranges));
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ });
+ }
+
+ long t1 = System.currentTimeMillis();
+
+ for (int i = 0; i < threads.length; i++) {
+ threads[i].start();
+ }
+
+ for (int i = 0; i < threads.length; i++) {
+ threads[i].join();
+ }
+
+ long t2 = System.currentTimeMillis();
+
+ System.out.printf("tt : %6.2f%n", (t2 - t1) / 1000.0);
+
+ } else {
+ throw new IllegalArgumentException();
+ }
+
+ }
+
+ private static ScanStats runScanTest(Connector connector, int numLoop, List<Range> ranges) throws Exception {
+ Scanner scanner = null;/*
+ * connector.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS); ColumnFQ.fetch(scanner,
+ * Constants.METADATA_LOCATION_COLUMN); ColumnFQ.fetch(scanner, Constants.METADATA_PREV_ROW_COLUMN);
+ */
+
+ BatchScanner bs = connector.createBatchScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS, 1);
+ bs.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
+ Constants.METADATA_PREV_ROW_COLUMN.fetch(bs);
+
+ bs.setRanges(ranges);
+
+ // System.out.println(ranges);
+
+ ScanStats stats = new ScanStats();
+ for (int i = 0; i < numLoop; i++) {
+ ScanStat ss = scan(bs, ranges, scanner);
+ stats.merge(ss);
+ }
+
+ return stats;
+ }
+
+ private static class ScanStat {
+ long delta1;
+ long delta2;
+ int count1;
+ int count2;
+ }
+
+ private static class ScanStats {
+ Stat delta1 = new Stat();
+ Stat delta2 = new Stat();
+ Stat count1 = new Stat();
+ Stat count2 = new Stat();
+
+ void merge(ScanStat ss) {
+ delta1.addStat(ss.delta1);
+ delta2.addStat(ss.delta2);
+ count1.addStat(ss.count1);
+ count2.addStat(ss.count2);
+ }
+
+ @Override
+ public String toString() {
+ return "[" + delta1 + "] [" + delta2 + "]";
+ }
+ }
+
+ private static ScanStat scan(BatchScanner bs, List<Range> ranges, Scanner scanner) {
+
+ // System.out.println("ranges : "+ranges);
+
+ ScanStat ss = new ScanStat();
+
+ long t1 = System.currentTimeMillis();
+ int count = 0;
+ for (@SuppressWarnings("unused")
+ Entry<Key,Value> entry : bs) {
+ count++;
+ }
+ long t2 = System.currentTimeMillis();
+
+ ss.delta1 = (t2 - t1);
+ ss.count1 = count;
+
+ count = 0;
+ t1 = System.currentTimeMillis();
+ /*
+ * for (Range range : ranges) { scanner.setRange(range); for (Entry<Key, Value> entry : scanner) { count++; } }
+ */
+
+ t2 = System.currentTimeMillis();
+
+ ss.delta2 = (t2 - t1);
+ ss.count2 = count;
+
+ return ss;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/performance/thrift/NullTserver.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/performance/thrift/NullTserver.java
index 71970d3,0000000..e6fcd5b
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/performance/thrift/NullTserver.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/thrift/NullTserver.java
@@@ -1,258 -1,0 +1,252 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.performance.thrift;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
- import org.apache.accumulo.trace.thrift.TInfo;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.thrift.InitialMultiScan;
+import org.apache.accumulo.core.data.thrift.InitialScan;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.data.thrift.MapFileInfo;
+import org.apache.accumulo.core.data.thrift.MultiScanResult;
+import org.apache.accumulo.core.data.thrift.ScanResult;
+import org.apache.accumulo.core.data.thrift.TColumn;
+import org.apache.accumulo.core.data.thrift.TConstraintViolationSummary;
+import org.apache.accumulo.core.data.thrift.TKeyExtent;
+import org.apache.accumulo.core.data.thrift.TMutation;
+import org.apache.accumulo.core.data.thrift.TRange;
+import org.apache.accumulo.core.data.thrift.UpdateErrors;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Processor;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.client.ClientServiceHandler;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.master.state.Assignment;
+import org.apache.accumulo.server.master.state.MetaDataStateStore;
+import org.apache.accumulo.server.master.state.MetaDataTableScanner;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletLocationState;
+import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.accumulo.server.util.TServerUtils;
+import org.apache.accumulo.server.zookeeper.TransactionWatcher;
++import org.apache.accumulo.trace.thrift.TInfo;
+import org.apache.hadoop.io.Text;
+import org.apache.thrift.TException;
+
+import com.beust.jcommander.Parameter;
+
+
+/**
+ * The purpose of this class is to server as fake tserver that is a data sink like /dev/null. NullTserver modifies the !METADATA location entries for a table to
+ * point to it. This allows thrift performance to be measured by running any client code that writes to a table.
+ *
+ */
+
+public class NullTserver {
+
+ public static class ThriftClientHandler extends ClientServiceHandler implements TabletClientService.Iface {
+
+ private long updateSession = 1;
+
+ public ThriftClientHandler(Instance instance, TransactionWatcher watcher) {
+ super(instance, watcher);
+ }
+
+ @Override
+ public long startUpdate(TInfo tinfo, TCredentials credentials) {
+ return updateSession++;
+ }
+
+ @Override
+ public void applyUpdates(TInfo tinfo, long updateID, TKeyExtent keyExtent, List<TMutation> mutation) {}
+
+ @Override
+ public UpdateErrors closeUpdate(TInfo tinfo, long updateID) {
+ return new UpdateErrors(new HashMap<TKeyExtent,Long>(), new ArrayList<TConstraintViolationSummary>(), new HashMap<TKeyExtent, SecurityErrorCode>());
+ }
+
+ @Override
+ public List<TKeyExtent> bulkImport(TInfo tinfo, TCredentials credentials, long tid, Map<TKeyExtent,Map<String,MapFileInfo>> files, boolean setTime) {
+ return null;
+ }
+
+ @Override
+ public void closeMultiScan(TInfo tinfo, long scanID) {}
+
+ @Override
+ public void closeScan(TInfo tinfo, long scanID) {}
+
+ @Override
+ public MultiScanResult continueMultiScan(TInfo tinfo, long scanID) {
+ return null;
+ }
+
+ @Override
+ public ScanResult continueScan(TInfo tinfo, long scanID) {
+ return null;
+ }
+
+ @Override
+ public void splitTablet(TInfo tinfo, TCredentials credentials, TKeyExtent extent, ByteBuffer splitPoint) {
+
+ }
+
+ @Override
+ public InitialMultiScan startMultiScan(TInfo tinfo, TCredentials credentials, Map<TKeyExtent,List<TRange>> batch, List<TColumn> columns,
+ List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites) {
+ return null;
+ }
+
+ @Override
+ public InitialScan startScan(TInfo tinfo, TCredentials credentials, TKeyExtent extent, TRange range, List<TColumn> columns, int batchSize,
+ List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites, boolean isolated) {
+ return null;
+ }
+
+ @Override
+ public void update(TInfo tinfo, TCredentials credentials, TKeyExtent keyExtent, TMutation mutation) {
+
+ }
+
+ @Override
+ public TabletServerStatus getTabletServerStatus(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ return null;
+ }
+
+ @Override
+ public List<TabletStats> getTabletStats(TInfo tinfo, TCredentials credentials, String tableId) throws ThriftSecurityException, TException {
+ return null;
+ }
+
+ @Override
+ public TabletStats getHistoricalStats(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ return null;
+ }
+
+ @Override
+ public void halt(TInfo tinfo, TCredentials credentials, String lock) throws ThriftSecurityException, TException {}
+
+ @Override
+ public void fastHalt(TInfo tinfo, TCredentials credentials, String lock) {}
+
+ @Override
+ public void loadTablet(TInfo tinfo, TCredentials credentials, String lock, TKeyExtent extent) throws TException {}
+
+ @Override
+ public void unloadTablet(TInfo tinfo, TCredentials credentials, String lock, TKeyExtent extent, boolean save) throws TException {}
+
+ @Override
+ public List<ActiveScan> getActiveScans(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ return new ArrayList<ActiveScan>();
+ }
+
+ @Override
+ public void chop(TInfo tinfo, TCredentials credentials, String lock, TKeyExtent extent) throws TException {}
+
+ @Override
+ public void flushTablet(TInfo tinfo, TCredentials credentials, String lock, TKeyExtent extent) throws TException {
+
+ }
+
+ @Override
+ public void compact(TInfo tinfo, TCredentials credentials, String lock, String tableId, ByteBuffer startRow, ByteBuffer endRow) throws TException {
+
+ }
+
+ @Override
+ public void flush(TInfo tinfo, TCredentials credentials, String lock, String tableId, ByteBuffer startRow, ByteBuffer endRow) throws TException {
+
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface#removeLogs(org.apache.accumulo.trace.thrift.TInfo,
- * org.apache.accumulo.core.security.thrift.Credentials, java.util.List)
- */
+ @Override
+ public void removeLogs(TInfo tinfo, TCredentials credentials, List<String> filenames) throws TException {
+ }
+
+ @Override
+ public List<ActiveCompaction> getActiveCompactions(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ return new ArrayList<ActiveCompaction>();
+ }
+ }
+
+ static class Opts extends Help {
+ @Parameter(names={"-i", "--instance"}, description="instance name", required=true)
+ String iname = null;
+ @Parameter(names={"-z", "--keepers"}, description="comma-separated list of zookeeper host:ports", required=true)
+ String keepers = null;
+ @Parameter(names="--table", description="table to adopt", required=true)
+ String tableName = null;
+ @Parameter(names="--port", description="port number to use")
+ int port = DefaultConfiguration.getInstance().getPort(Property.TSERV_CLIENTPORT);
+ }
+
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(NullTserver.class.getName(), args);
+
+ TransactionWatcher watcher = new TransactionWatcher();
+ ThriftClientHandler tch = new ThriftClientHandler(HdfsZooInstance.getInstance(), watcher);
+ Processor<Iface> processor = new Processor<Iface>(tch);
+ TServerUtils.startTServer(opts.port, processor, "NullTServer", "null tserver", 2, 1000, 10*1024*1024);
+
+ InetSocketAddress addr = new InetSocketAddress(InetAddress.getLocalHost(), opts.port);
+
+ // modify !METADATA
+ ZooKeeperInstance zki = new ZooKeeperInstance(opts.iname, opts.keepers);
+ String tableId = Tables.getTableId(zki, opts.tableName);
+
+ // read the locations for the table
+ Range tableRange = new KeyExtent(new Text(tableId), null, null).toMetadataRange();
+ MetaDataTableScanner s = new MetaDataTableScanner(zki, SecurityConstants.getSystemCredentials(), tableRange);
+ long randomSessionID = opts.port;
+ TServerInstance instance = new TServerInstance(addr, randomSessionID);
+ List<Assignment> assignments = new ArrayList<Assignment>();
+ while (s.hasNext()) {
+ TabletLocationState next = s.next();
+ assignments.add(new Assignment(next.extent, instance));
+ }
+ s.close();
+ // point them to this server
+ MetaDataStateStore store = new MetaDataStateStore();
+ store.setLocations(assignments);
+
+ while (true) {
+ UtilWaitThread.sleep(10000);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/randomwalk/Framework.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/Framework.java
index 9d01929,0000000..7cb58c9
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Framework.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Framework.java
@@@ -1,129 -1,0 +1,126 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.randomwalk;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.util.HashMap;
+import java.util.Properties;
+
+import org.apache.log4j.Logger;
+import org.apache.log4j.xml.DOMConfigurator;
+
+import com.beust.jcommander.Parameter;
+
+public class Framework {
+
+ private static final Logger log = Logger.getLogger(Framework.class);
+ private HashMap<String,Node> nodes = new HashMap<String,Node>();
+ private String configDir = null;
+ private static final Framework INSTANCE = new Framework();
+
+ /**
+ * @return Singleton instance of Framework
+ */
+ public static Framework getInstance() {
+ return INSTANCE;
+ }
+
+ public String getConfigDir() {
+ return configDir;
+ }
+
+ public void setConfigDir(String confDir) {
+ configDir = confDir;
+ }
+
+ /**
+ * Run random walk framework
+ *
+ * @param startName
+ * Full name of starting graph or test
- * @param state
- * @param confDir
+ */
+ public int run(String startName, State state, String confDir) {
+
+ try {
+ System.out.println("confDir " + confDir);
+ setConfigDir(confDir);
+ Node node = getNode(startName);
+ node.visit(state, new Properties());
+ } catch (Exception e) {
+ log.error("Error during random walk", e);
+ return -1;
+ }
+ return 0;
+ }
+
+ /**
+ * Creates node (if it does not already exist) and inserts into map
+ *
+ * @param id
+ * Name of node
+ * @return Node specified by id
- * @throws Exception
+ */
+ public Node getNode(String id) throws Exception {
+
+ // check for node in nodes
+ if (nodes.containsKey(id)) {
+ return nodes.get(id);
+ }
+
+ // otherwise create and put in nodes
+ Node node = null;
+ if (id.endsWith(".xml")) {
+ node = new Module(new File(configDir + "modules/" + id));
+ } else {
+ node = (Test) Class.forName(id).newInstance();
+ }
+ nodes.put(id, node);
+ return node;
+ }
+
+ static class Opts extends org.apache.accumulo.core.cli.Help {
+ @Parameter(names="--configDir", required=true, description="directory containing the test configuration")
+ String configDir;
+ @Parameter(names="--logDir", required=true, description="location of the local logging directory")
+ String localLogPath;
+ @Parameter(names="--logId", required=true, description="a unique log identifier (like a hostname, or pid)")
+ String logId;
+ @Parameter(names="--module", required=true, description="the name of the module to run")
+ String module;
+ }
+
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(Framework.class.getName(), args);
+
+ Properties props = new Properties();
+ FileInputStream fis = new FileInputStream(opts.configDir + "/randomwalk.conf");
+ props.load(fis);
+ fis.close();
+
+ System.setProperty("localLog", opts.localLogPath + "/" + opts.logId);
+ System.setProperty("nfsLog", props.getProperty("NFS_LOGPATH") + "/" + opts.logId);
+
+ DOMConfigurator.configure(opts.configDir + "logger.xml");
+
+ State state = new State(props);
+ int retval = getInstance().run(opts.module, state, opts.configDir);
+
+ System.exit(retval);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
index 1868ade,0000000..b74b6cd
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
@@@ -1,64 -1,0 +1,63 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.randomwalk;
+
+import java.util.Properties;
+
+import org.apache.log4j.Logger;
+
+/**
+ * Represents a point in graph of RandomFramework
+ */
+public abstract class Node {
+
+ protected final Logger log = Logger.getLogger(this.getClass());
+ long progress = System.currentTimeMillis();
+
+ /**
+ * Visits node
+ *
+ * @param state
+ * Random walk state passed between nodes
- * @throws Exception
+ */
+ public abstract void visit(State state, Properties props) throws Exception;
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null)
+ return false;
+ return toString().equals(o.toString());
+ }
+
+ @Override
+ public String toString() {
+ return this.getClass().getName();
+ }
+
+ @Override
+ public int hashCode() {
+ return toString().hashCode();
+ }
+
+ synchronized public void makingProgress() {
+ progress = System.currentTimeMillis();
+ }
+
+ synchronized public long lastProgress() {
+ return progress;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
index a0dd37c,0000000..4581b04
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
@@@ -1,110 -1,0 +1,107 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.randomwalk.concurrent;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.test.randomwalk.State;
+import org.apache.accumulo.test.randomwalk.Test;
+
+/**
+ *
+ */
+public class CheckBalance extends Test {
+
+ static final String LAST_UNBALANCED_TIME = "lastUnbalancedTime";
+ static final String UNBALANCED_COUNT = "unbalancedCount";
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.test.randomwalk.Node#visit(org.apache.accumulo.test.randomwalk.State, java.util.Properties)
- */
+ @Override
+ public void visit(State state, Properties props) throws Exception {
+ log.debug("checking balance");
+ Map<String,Long> counts = new HashMap<String,Long>();
+ Scanner scanner = state.getConnector().createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ scanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
+ for (Entry<Key,Value> entry : scanner) {
+ String location = entry.getKey().getColumnQualifier().toString();
+ Long count = counts.get(location);
+ if (count == null)
+ count = Long.valueOf(0);
+ counts.put(location, count + 1);
+ }
+ double total = 0.;
+ for (Long count : counts.values()) {
+ total += count.longValue();
+ }
+ final double average = total / counts.size();
+ final double sd = stddev(counts.values(), average);
+ log.debug("average " + average + ", standard deviation " + sd);
+
+ // Check for balanced # of tablets on each node
+ double maxDifference = 2.0 * sd;
+ String unbalancedLocation = null;
+ long lastCount = 0L;
+ boolean balanced = true;
+ for (Entry<String,Long> entry : counts.entrySet()) {
+ long thisCount = entry.getValue().longValue();
+ if (Math.abs(thisCount - average) > maxDifference) {
+ balanced = false;
+ log.debug("unbalanced: " + entry.getKey() + " has " + entry.getValue() + " tablets and the average is " + average);
+ unbalancedLocation = entry.getKey();
+ lastCount = thisCount;
+ }
+ }
+
+ // It is expected that the number of tablets will be uneven for short
+ // periods of time. Don't complain unless we've seen it only unbalanced
+ // over a 15 minute period and it's been at least three checks.
+ if (!balanced) {
+ Long last = state.getLong(LAST_UNBALANCED_TIME);
+ if (last != null && System.currentTimeMillis() - last > 15 * 60 * 1000) {
+ Integer count = state.getInteger(UNBALANCED_COUNT);
+ if (count == null)
+ count = Integer.valueOf(0);
+ if (count > 3)
+ throw new Exception("servers are unbalanced! location " + unbalancedLocation + " count " + lastCount + " too far from average " + average);
+ count++;
+ state.set(UNBALANCED_COUNT, count);
+ }
+ if (last == null)
+ state.set(LAST_UNBALANCED_TIME, System.currentTimeMillis());
+ } else {
+ state.remove(LAST_UNBALANCED_TIME);
+ state.remove(UNBALANCED_COUNT);
+ }
+ }
+
+ private static double stddev(Collection<Long> samples, double avg) {
+ int num = samples.size();
+ double sqrtotal = 0.0;
+ for (Long s : samples) {
+ double diff = s.doubleValue() - avg;
+ sqrtotal += diff * diff;
+ }
- return Math.sqrt(sqrtotal / (double) num);
++ return Math.sqrt(sqrtotal / num);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
----------------------------------------------------------------------
diff --cc trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
index 049b2a2,0000000..dfa9f0c
mode 100644,000000..100644
--- a/trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
+++ b/trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
@@@ -1,132 -1,0 +1,122 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.trace.instrument.receivers;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.ZooKeeper.States;
+
+/**
+ * Find a Span collector via zookeeper and push spans there via Thrift RPC
+ *
+ */
+public class ZooSpanClient extends SendSpansViaThrift {
+
+ private static final Logger log = Logger.getLogger(ZooSpanClient.class);
+ private static final int TOTAL_TIME_WAIT_CONNECT_MS = 10 * 1000;
+ private static final int TIME_WAIT_CONNECT_CHECK_MS = 100;
+ private static final Charset UTF8 = Charset.forName("UTF-8");
+
+ ZooKeeper zoo = null;
+ final String path;
+ final Random random = new Random();
+ final List<String> hosts = new ArrayList<String>();
+
+ public ZooSpanClient(String keepers, final String path, String host, String service, long millis) throws IOException, KeeperException, InterruptedException {
+ super(host, service, millis);
+ this.path = path;
+ zoo = new ZooKeeper(keepers, 30 * 1000, new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ try {
+ if (zoo != null) {
+ updateHosts(path, zoo.getChildren(path, null));
+ }
+ } catch (Exception ex) {
+ log.error("unable to get destination hosts in zookeeper", ex);
+ }
+ }
+ });
+ for (int i = 0; i < TOTAL_TIME_WAIT_CONNECT_MS; i += TIME_WAIT_CONNECT_CHECK_MS) {
+ if (zoo.getState().equals(States.CONNECTED))
+ break;
+ try {
+ Thread.sleep(TIME_WAIT_CONNECT_CHECK_MS);
+ } catch (InterruptedException ex) {
+ break;
+ }
+ }
+ zoo.getChildren(path, true);
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see trace.instrument.receivers.AsyncSpanReceiver#flush()
- */
+ @Override
+ public void flush() {
+ if (!hosts.isEmpty())
+ super.flush();
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see trace.instrument.receivers.AsyncSpanReceiver#sendSpans()
- */
+ @Override
+ void sendSpans() {
+ if (hosts.isEmpty()) {
+ if (!sendQueue.isEmpty()) {
+ log.error("No hosts to send data to, dropping queued spans");
+ synchronized (sendQueue) {
+ sendQueue.clear();
+ sendQueue.notifyAll();
+ }
+ }
+ } else {
+ super.sendSpans();
+ }
+ }
+
+ synchronized private void updateHosts(String path, List<String> children) {
+ log.debug("Scanning trace hosts in zookeeper: " + path);
+ try {
+ List<String> hosts = new ArrayList<String>();
+ for (String child : children) {
+ byte[] data = zoo.getData(path + "/" + child, null, null);
+ hosts.add(new String(data, UTF8));
+ }
+ this.hosts.clear();
+ this.hosts.addAll(hosts);
+ log.debug("Trace hosts: " + this.hosts);
+ } catch (Exception ex) {
+ log.error("unable to get destination hosts in zookeeper", ex);
+ }
+ }
+
+ @Override
+ synchronized protected String getSpanKey(Map<String,String> data) {
+ if (hosts.size() > 0) {
+ String host = hosts.get(random.nextInt(hosts.size()));
+ log.debug("sending data to " + host);
+ return host;
+ }
+ return null;
+ }
+}
[61/64] [abbrv] Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleParameters.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleParameters.java
index 5ae072a,0000000..244a877
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleParameters.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleParameters.java
@@@ -1,643 -1,0 +1,637 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.core.security.crypto;
+
+import java.io.FilterOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.security.SecureRandom;
+import java.util.Map;
+
+import javax.crypto.Cipher;
+import javax.crypto.CipherOutputStream;
+
+/**
+ * This class defines several parameters needed by by a module providing cryptographic stream support in Accumulo. The following Javadoc details which
+ * parameters are used for which operations (encryption vs. decryption), which ones return values (i.e. are "out" parameters from the {@link CryptoModule}), and
+ * which ones are required versus optional in certain situations.
+ *
+ * Most of the time, these classes can be constructed using
+ * {@link CryptoModuleFactory#createParamsObjectFromAccumuloConfiguration(org.apache.accumulo.core.conf.AccumuloConfiguration)}.
+ */
+public class CryptoModuleParameters {
+
+ /**
+ * Gets the name of the symmetric algorithm to use for encryption.
+ *
+ * @see CryptoModuleParameters#setAlgorithmName(String)
+ */
+
+ public String getAlgorithmName() {
+ return algorithmName;
+ }
+
+ /**
+ * Sets the name of the symmetric algorithm to use for an encryption stream.
+ * <p>
+ * Valid names are names recognized by your cryptographic engine provider. For the default Java provider, valid names would include things like "AES", "RC4",
+ * "DESede", etc.
+ * <p>
+ * For <b>encryption</b>, this value is <b>required</b> and is always used. Its value should be prepended or otherwise included with the ciphertext for future
+ * decryption. <br>
+ * For <b>decryption</b>, this value is often disregarded in favor of the value encoded with the ciphertext.
+ *
+ * @param algorithmName
+ * the name of the cryptographic algorithm to use.
+ * @see <a href="http://docs.oracle.com/javase/1.5.0/docs/guide/security/jce/JCERefGuide.html#AppA">Standard Algorithm Names in JCE</a>
+ *
+ */
+
+ public void setAlgorithmName(String algorithmName) {
+ this.algorithmName = algorithmName;
+ }
+
+ /**
+ * Gets the name of the encryption mode to use for encryption.
+ *
+ * @see CryptoModuleParameters#setEncryptionMode(String)
+ */
+
+ public String getEncryptionMode() {
+ return encryptionMode;
+ }
+
+ /**
+ * Sets the name of the encryption mode to use for an encryption stream.
+ * <p>
+ * Valid names are names recognized by your cryptographic engine provider. For the default Java provider, valid names would include things like "EBC", "CBC",
+ * "CFB", etc.
+ * <p>
+ * For <b>encryption</b>, this value is <b>required</b> and is always used. Its value should be prepended or otherwise included with the ciphertext for future
+ * decryption. <br>
+ * For <b>decryption</b>, this value is often disregarded in favor of the value encoded with the ciphertext.
+ *
+ * @param encryptionMode
+ * the name of the encryption mode to use.
+ * @see <a href="http://docs.oracle.com/javase/1.5.0/docs/guide/security/jce/JCERefGuide.html#AppA">Standard Mode Names in JCE</a>
+ *
+ */
+
+ public void setEncryptionMode(String encryptionMode) {
+ this.encryptionMode = encryptionMode;
+ }
+
+ /**
+ * Gets the name of the padding type to use for encryption.
+ *
+ * @see CryptoModuleParameters#setPadding(String)
+ */
+
+ public String getPadding() {
+ return padding;
+ }
+
+ /**
+ * Sets the name of the padding type to use for an encryption stream.
+ * <p>
+ * Valid names are names recognized by your cryptographic engine provider. For the default Java provider, valid names would include things like "NoPadding",
+ * "None", etc.
+ * <p>
+ * For <b>encryption</b>, this value is <b>required</b> and is always used. Its value should be prepended or otherwise included with the ciphertext for future
+ * decryption. <br>
+ * For <b>decryption</b>, this value is often disregarded in favor of the value encoded with the ciphertext.
+ *
+ * @param padding
+ * the name of the padding type to use.
+ * @see <a href="http://docs.oracle.com/javase/1.5.0/docs/guide/security/jce/JCERefGuide.html#AppA">Standard Padding Names in JCE</a>
+ *
+ */
+ public void setPadding(String padding) {
+ this.padding = padding;
+ }
+
+ /**
+ * Gets the plaintext secret key.
+ * <p>
+ * For <b>decryption</b>, this value is often the out parameter of using a secret key encryption strategy to decrypt an encrypted version of this secret key.
+ * (See {@link CryptoModuleParameters#setKeyEncryptionStrategyClass(String)}.)
+ *
+ *
+ * @see CryptoModuleParameters#setPlaintextKey(byte[])
+ */
+ public byte[] getPlaintextKey() {
+ return plaintextKey;
+ }
+
+ /**
+ * Sets the plaintext secret key that will be used to encrypt and decrypt bytes.
+ * <p>
+ * Valid values and lengths for this secret key depend entirely on the algorithm type. Refer to the documentation about the algorithm for further information.
+ * <p>
+ * For <b>encryption</b>, this value is <b>optional</b>. If it is not provided, it will be automatically generated by the underlying cryptographic module. <br>
+ * For <b>decryption</b>, this value is often obtained from the underlying cipher stream, or derived from the encrypted version of the key (see
+ * {@link CryptoModuleParameters#setEncryptedKey(byte[])}).
+ *
+ * @param plaintextKey
+ * the value of the plaintext secret key
+ */
+
+ public void setPlaintextKey(byte[] plaintextKey) {
+ this.plaintextKey = plaintextKey;
+ }
+
+ /**
+ * Gets the length of the secret key.
+ *
+ * @see CryptoModuleParameters#setKeyLength(int)
+ */
+ public int getKeyLength() {
+ return keyLength;
+ }
+
+ /**
+ * Sets the length of the secret key that will be used to encrypt and decrypt bytes.
+ * <p>
+ * Valid lengths depend entirely on the algorithm type. Refer to the documentation about the algorithm for further information. (For example, AES may use
+ * either 128 or 256 bit keys in the default Java cryptography provider.)
+ * <p>
+ * For <b>encryption</b>, this value is <b>required if the secret key is not set</b>. <br>
+ * For <b>decryption</b>, this value is often obtained from the underlying cipher stream, or derived from the encrypted version of the key (see
+ * {@link CryptoModuleParameters#setEncryptedKey(byte[])}).
+ *
+ * @param keyLength
+ * the length of the secret key to be generated
+ */
+
+ public void setKeyLength(int keyLength) {
+ this.keyLength = keyLength;
+ }
+
+ /**
+ * Gets the random number generator name.
+ *
+ * @see CryptoModuleParameters#setRandomNumberGenerator(String)
+ */
+
+ public String getRandomNumberGenerator() {
+ return randomNumberGenerator;
+ }
+
+ /**
+ * Sets the name of the random number generator to use. The default for this for the baseline JCE implementation is "SHA1PRNG".
+ * <p>
+ *
+ * <p>
+ * For <b>encryption</b>, this value is <b>required</b>. <br>
+ * For <b>decryption</b>, this value is often obtained from the underlying cipher stream.
+ *
+ * @param randomNumberGenerator
+ * the name of the random number generator to use
+ */
+
+ public void setRandomNumberGenerator(String randomNumberGenerator) {
+ this.randomNumberGenerator = randomNumberGenerator;
+ }
+
+ /**
+ * Gets the random number generator provider name.
+ *
+ * @see CryptoModuleParameters#setRandomNumberGeneratorProvider(String)
+ */
+ public String getRandomNumberGeneratorProvider() {
+ return randomNumberGeneratorProvider;
+ }
+
+ /**
+ * Sets the name of the random number generator provider to use. The default for this for the baseline JCE implementation is "SUN".
+ * <p>
+ * The provider, as the name implies, provides the RNG implementation specified by {@link CryptoModuleParameters#getRandomNumberGenerator()}.
+ * <p>
+ * For <b>encryption</b>, this value is <b>required</b>. <br>
+ * For <b>decryption</b>, this value is often obtained from the underlying cipher stream.
+ *
+ * @param randomNumberGeneratorProvider
+ * the name of the provider to use
+ */
+
+ public void setRandomNumberGeneratorProvider(String randomNumberGeneratorProvider) {
+ this.randomNumberGeneratorProvider = randomNumberGeneratorProvider;
+ }
+
+ /**
+ * Gets the key encryption strategy class.
+ *
+ * @see CryptoModuleParameters#setKeyEncryptionStrategyClass(String)
+ */
+
+ public String getKeyEncryptionStrategyClass() {
+ return keyEncryptionStrategyClass;
+ }
+
+ /**
+ * Sets the class name of the key encryption strategy class. The class obeys the {@link SecretKeyEncryptionStrategy} interface. It instructs the
+ * {@link DefaultCryptoModule} on how to encrypt the keys it uses to secure the streams.
+ * <p>
+ * The default implementation of this interface, {@link CachingHDFSSecretKeyEncryptionStrategy}, creates a random key encryption key (KEK) as another symmetric
+ * key and places the KEK into HDFS. <i>This is not really very secure.</i> Users of the crypto modules are encouraged to either safeguard that KEK carefully
+ * or to obtain and use another {@link SecretKeyEncryptionStrategy} class.
+ * <p>
+ * For <b>encryption</b>, this value is <b>optional</b>. If it is not specified, then it assumed that the secret keys used for encrypting files will not be
+ * encrypted. This is not a secure approach, thus setting this is highly recommended.<br>
+ * For <b>decryption</b>, this value is often obtained from the underlying cipher stream. However, the underlying stream's value can be overridden (at least
+ * when using {@link DefaultCryptoModule}) by setting the {@link CryptoModuleParameters#setOverrideStreamsSecretKeyEncryptionStrategy(boolean)} to true.
+ *
+ * @param keyEncryptionStrategyClass
+ * the name of the key encryption strategy class to use
+ */
+ public void setKeyEncryptionStrategyClass(String keyEncryptionStrategyClass) {
+ this.keyEncryptionStrategyClass = keyEncryptionStrategyClass;
+ }
+
+ /**
+ * Gets the encrypted version of the plaintext key. This parameter is generally either obtained from an underlying stream or computed in the process of
+ * employed the {@link CryptoModuleParameters#getKeyEncryptionStrategyClass()}.
+ *
+ * @see CryptoModuleParameters#setEncryptedKey(byte[])
+ */
+ public byte[] getEncryptedKey() {
+ return encryptedKey;
+ }
+
+ /**
+ * Sets the encrypted version of the plaintext key ({@link CryptoModuleParameters#getPlaintextKey()}). Generally this operation will be done either by:
+ * <p>
+ * <ul>
+ * <li>the code reading an encrypted stream and coming across the encrypted version of one of these keys, OR
+ * <li>the {@link CryptoModuleParameters#getKeyEncryptionStrategyClass()} that encrypted the plaintext key (see
+ * {@link CryptoModuleParameters#getPlaintextKey()}).
+ * <ul>
+ * <p>
+ * For <b>encryption</b>, this value is generally not required, but is usually set by the underlying module during encryption. <br>
+ * For <b>decryption</b>, this value is <b>usually required</b>.
+ *
+ *
+ * @param encryptedKey
+ * the encrypted value of the plaintext key
+ */
+
+ public void setEncryptedKey(byte[] encryptedKey) {
+ this.encryptedKey = encryptedKey;
+ }
+
+ /**
+ * Gets the opaque ID associated with the encrypted version of the plaintext key.
+ *
+ * @see CryptoModuleParameters#setOpaqueKeyEncryptionKeyID(String)
+ */
+ public String getOpaqueKeyEncryptionKeyID() {
+ return opaqueKeyEncryptionKeyID;
+ }
+
+ /**
+ * Sets an opaque ID assocaited with the encrypted version of the plaintext key.
+ * <p>
+ * Often, implementors of the {@link SecretKeyEncryptionStrategy} will need to record some information about how they encrypted a particular plaintext key.
+ * For example, if the strategy employs several keys for its encryption, it will want to record which key it used. The caller should not have to worry about
+ * the format or contents of this internal ID; thus, the strategy class will encode whatever information it needs into this string. It is then beholden to the
+ * calling code to record this opqaue string properly to the underlying cryptographically-encoded stream, and then set the opaque ID back into this parameter
+ * object upon reading.
+ * <p>
+ * For <b>encryption</b>, this value is generally not required, but will be typically generated and set by the {@link SecretKeyEncryptionStrategy} class (see
+ * {@link CryptoModuleParameters#getKeyEncryptionStrategyClass()}). <br>
+ * For <b>decryption</b>, this value is <b>required</b>, though it will typically be read from the underlying stream.
+ *
+ * @param opaqueKeyEncryptionKeyID
+ * the opaque ID assoicated with the encrypted version of the plaintext key (see {@link CryptoModuleParameters#getEncryptedKey()}).
+ */
+
+ public void setOpaqueKeyEncryptionKeyID(String opaqueKeyEncryptionKeyID) {
+ this.opaqueKeyEncryptionKeyID = opaqueKeyEncryptionKeyID;
+ }
+
+ /**
+ * Gets the flag that indicates whether or not the module should record its cryptographic parameters to the stream automatically, or rely on the calling code
+ * to do so.
+ *
+ * @see CryptoModuleParameters#setRecordParametersToStream(boolean)
+ */
+ public boolean getRecordParametersToStream() {
+ return recordParametersToStream;
+ }
+
+ /**
+ * Gets the flag that indicates whether or not the module should record its cryptographic parameters to the stream automatically, or rely on the calling code
+ * to do so.
+ *
+ * <p>
+ *
+ * If this is set to <i>true</i>, then the stream passed to {@link CryptoModule#getEncryptingOutputStream(CryptoModuleParameters)} will be <i>written to by
+ * the module</i> before it is returned to the caller. There are situations where it is easier to let the crypto module do this writing on behalf of the
+ * caller, and other times where it is not appropriate (if the format of the underlying stream must be carefully maintained, for instance).
+ *
+ * @param recordParametersToStream
+ * whether or not to require the module to record its parameters to the stream by itself
+ */
+ public void setRecordParametersToStream(boolean recordParametersToStream) {
+ this.recordParametersToStream = recordParametersToStream;
+ }
+
+ /**
+ * Gets the flag that indicates whether or not to close the underlying stream when the cipher stream is closed.
+ *
+ * @see CryptoModuleParameters#setCloseUnderylingStreamAfterCryptoStreamClose(boolean)
+ */
+ public boolean getCloseUnderylingStreamAfterCryptoStreamClose() {
+ return closeUnderylingStreamAfterCryptoStreamClose;
+ }
+
+ /**
+ * Sets the flag that indicates whether or not to close the underlying stream when the cipher stream is closed.
+ *
+ * <p>
+ *
+ * {@link CipherOutputStream} will only output its padding bytes when its {@link CipherOutputStream#close()} method is called. However, there are times when a
+ * caller doesn't want its underlying stream closed at the time that the {@link CipherOutputStream} is closed. This flag indicates that the
+ * {@link CryptoModule} should wrap the underlying stream in a basic {@link FilterOutputStream} which will swallow any close() calls and prevent them from
+ * propogating to the underlying stream.
+ *
+ * @param closeUnderylingStreamAfterCryptoStreamClose
+ * the flag that indicates whether or not to close the underlying stream when the cipher stream is closed
+ */
+ public void setCloseUnderylingStreamAfterCryptoStreamClose(boolean closeUnderylingStreamAfterCryptoStreamClose) {
+ this.closeUnderylingStreamAfterCryptoStreamClose = closeUnderylingStreamAfterCryptoStreamClose;
+ }
+
+ /**
+ * Gets the flag that indicates if the underlying stream's key encryption strategy should be overridden by the currently configured key encryption strategy.
+ *
+ * @see CryptoModuleParameters#setOverrideStreamsSecretKeyEncryptionStrategy(boolean)
+ */
+ public boolean getOverrideStreamsSecretKeyEncryptionStrategy() {
+ return overrideStreamsSecretKeyEncryptionStrategy;
+ }
+
+ /**
+ * Sets the flag that indicates if the underlying stream's key encryption strategy should be overridden by the currently configured key encryption strategy.
+ *
+ * <p>
+ *
+ * So, why is this important? Say you started out with the default secret key encryption strategy. So, now you have a secret key in HDFS that encrypts all the
+ * other secret keys. <i>Then</i> you deploy a key management solution. You want to move that secret key up to the key management server. Great! No problem.
+ * Except, all your encrypted files now contain a setting that says
+ * "hey I was encrypted by the default strategy, so find decrypt my key using that, not the key management server". This setting signals the
+ * {@link CryptoModule} that it should ignore the setting in the file and prefer the one from the configuration.
+ *
+ * @param overrideStreamsSecretKeyEncryptionStrategy
+ * the flag that indicates if the underlying stream's key encryption strategy should be overridden by the currently configured key encryption
+ * strategy
+ */
+
+ public void setOverrideStreamsSecretKeyEncryptionStrategy(boolean overrideStreamsSecretKeyEncryptionStrategy) {
+ this.overrideStreamsSecretKeyEncryptionStrategy = overrideStreamsSecretKeyEncryptionStrategy;
+ }
+
+ /**
+ * Gets the plaintext output stream to wrap for encryption.
+ *
+ * @see CryptoModuleParameters#setPlaintextOutputStream(OutputStream)
+ */
+ public OutputStream getPlaintextOutputStream() {
+ return plaintextOutputStream;
+ }
+
+ /**
+ * Sets the plaintext output stream to wrap for encryption.
+ *
+ * <p>
+ *
+ * For <b>encryption</b>, this parameter is <b>required</b>. <br>
+ * For <b>decryption</b>, this parameter is ignored.
- *
- * @param plaintextOutputStream
+ */
+ public void setPlaintextOutputStream(OutputStream plaintextOutputStream) {
+ this.plaintextOutputStream = plaintextOutputStream;
+ }
+
+ /**
+ * Gets the encrypted output stream, which is nearly always a wrapped version of the output stream from
+ * {@link CryptoModuleParameters#getPlaintextOutputStream()}.
+ *
+ * <p>
+ *
+ * Generally this method is used by {@link CryptoModule} classes as an <i>out</i> parameter from calling
+ * {@link CryptoModule#getEncryptingOutputStream(CryptoModuleParameters)}.
+ *
+ * @see CryptoModuleParameters#setEncryptedOutputStream(OutputStream)
+ */
+
+ public OutputStream getEncryptedOutputStream() {
+ return encryptedOutputStream;
+ }
+
+ /**
+ * Sets the encrypted output stream. This method should really only be called by {@link CryptoModule} implementations unless something very unusual is going
+ * on.
+ *
+ * @param encryptedOutputStream
+ * the encrypted version of the stream from output stream from {@link CryptoModuleParameters#getPlaintextOutputStream()}.
+ */
+ public void setEncryptedOutputStream(OutputStream encryptedOutputStream) {
+ this.encryptedOutputStream = encryptedOutputStream;
+ }
+
+ /**
+ * Gets the plaintext input stream, which is nearly always a wrapped version of the output from {@link CryptoModuleParameters#getEncryptedInputStream()}.
+ *
+ * <p>
+ *
+ * Generally this method is used by {@link CryptoModule} classes as an <i>out</i> parameter from calling
+ * {@link CryptoModule#getDecryptingInputStream(CryptoModuleParameters)}.
+ *
+ *
+ * @see CryptoModuleParameters#setPlaintextInputStream(InputStream)
+ */
+ public InputStream getPlaintextInputStream() {
+ return plaintextInputStream;
+ }
+
+ /**
+ * Sets the plaintext input stream, which is nearly always a wrapped version of the output from {@link CryptoModuleParameters#getEncryptedInputStream()}.
+ *
+ * <p>
+ *
+ * This method should really only be called by {@link CryptoModule} implementations.
- *
- * @param plaintextInputStream
+ */
+
+ public void setPlaintextInputStream(InputStream plaintextInputStream) {
+ this.plaintextInputStream = plaintextInputStream;
+ }
+
+ /**
+ * Gets the encrypted input stream to wrap for decryption.
+ *
+ * @see CryptoModuleParameters#setEncryptedInputStream(InputStream)
+ */
+ public InputStream getEncryptedInputStream() {
+ return encryptedInputStream;
+ }
+
+ /**
+ * Sets the encrypted input stream to wrap for decryption.
- *
- * @param encryptedInputStream
+ */
+
+ public void setEncryptedInputStream(InputStream encryptedInputStream) {
+ this.encryptedInputStream = encryptedInputStream;
+ }
+
+ /**
+ * Gets the initialized cipher object.
+ *
+ *
+ * @see CryptoModuleParameters#setCipher(Cipher)
+ */
+ public Cipher getCipher() {
+ return cipher;
+ }
+
+ /**
+ * Sets the initialized cipher object. Generally speaking, callers do not have to create and set this object. There may be circumstances where the cipher
+ * object is created outside of the module (to determine IV lengths, for one). If it is created and you want the module to use the cipher you already
+ * initialized, set it here.
+ *
+ * @param cipher
+ * the cipher object
+ */
+ public void setCipher(Cipher cipher) {
+ this.cipher = cipher;
+ }
+
+ /**
+ * Gets the initialized secure random object.
+ *
+ * @see CryptoModuleParameters#setSecureRandom(SecureRandom)
+ */
+ public SecureRandom getSecureRandom() {
+ return secureRandom;
+ }
+
+ /**
+ * Sets the initialized secure random object. Generally speaking, callers do not have to create and set this object. There may be circumstances where the
+ * random object is created outside of the module (for instance, to create a random secret key). If it is created outside the module and you want the module
+ * to use the random object you already created, set it here.
+ *
+ * @param secureRandom
+ * the {@link SecureRandom} object
+ */
+
+ public void setSecureRandom(SecureRandom secureRandom) {
+ this.secureRandom = secureRandom;
+ }
+
+ /**
+ * Gets the initialization vector to use for this crypto module.
+ *
+ * @see CryptoModuleParameters#setInitializationVector(byte[])
+ */
+ public byte[] getInitializationVector() {
+ return initializationVector;
+ }
+
+ /**
+ * Sets the initialization vector to use for this crypto module.
+ *
+ * <p>
+ *
+ * For <b>encryption</b>, this parameter is <i>optional</i>. If the initialization vector is created by the caller, for whatever reasons, it can be set here
+ * and the crypto module will use it. <br>
+ *
+ * For <b>decryption</b>, this parameter is <b>required</b>. It should be read from the underlying stream that contains the encrypted data.
+ *
+ * @param initializationVector
+ * the initialization vector to use for this crypto operation.
+ */
+ public void setInitializationVector(byte[] initializationVector) {
+ this.initializationVector = initializationVector;
+ }
+
+ /**
+ * Gets the size of the buffering stream that sits above the cipher stream
+ */
+ public int getBlockStreamSize() {
+ return blockStreamSize;
+ }
+
+ /**
+ * Sets the size of the buffering stream that sits above the cipher stream
+ */
+ public void setBlockStreamSize(int blockStreamSize) {
+ this.blockStreamSize = blockStreamSize;
+ }
+
+ /**
+ * Gets the overall set of options for the {@link CryptoModule}.
+ *
+ * @see CryptoModuleParameters#setAllOptions(Map)
+ */
+ public Map<String,String> getAllOptions() {
+ return allOptions;
+ }
+
+ /**
+ * Sets the overall set of options for the {@link CryptoModule}.
+ *
+ * <p>
+ *
+ * Often, options for the cryptographic modules will be encoded as key/value pairs in a configuration file. This map represents those values. It may include
+ * some of the parameters already called out as members of this class. It may contain any number of additional parameters which may be required by different
+ * module or key encryption strategy implementations.
+ *
+ * @param allOptions
+ * the set of key/value pairs that confiure a module, based on a configuration file
+ */
+ public void setAllOptions(Map<String,String> allOptions) {
+ this.allOptions = allOptions;
+ }
+
+ private String algorithmName = null;
+ private String encryptionMode = null;
+ private String padding = null;
+ private byte[] plaintextKey;
+ private int keyLength = 0;
+ private String randomNumberGenerator = null;
+ private String randomNumberGeneratorProvider = null;
+
+ private String keyEncryptionStrategyClass;
+ private byte[] encryptedKey;
+ private String opaqueKeyEncryptionKeyID;
+
+ private boolean recordParametersToStream = true;
+ private boolean closeUnderylingStreamAfterCryptoStreamClose = true;
+ private boolean overrideStreamsSecretKeyEncryptionStrategy = false;
+
+ private OutputStream plaintextOutputStream;
+ private OutputStream encryptedOutputStream;
+ private InputStream plaintextInputStream;
+ private InputStream encryptedInputStream;
+
+ private Cipher cipher;
+ private SecureRandom secureRandom;
+ private byte[] initializationVector;
+
+ private Map<String,String> allOptions;
+ private int blockStreamSize;
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/test/java/org/apache/accumulo/core/client/lexicoder/ReverseLexicoderTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/lexicoder/ReverseLexicoderTest.java
index cc363c7,0000000..e6bfca8
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/lexicoder/ReverseLexicoderTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/lexicoder/ReverseLexicoderTest.java
@@@ -1,62 -1,0 +1,60 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.lexicoder;
+
+import java.io.UnsupportedEncodingException;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+
+import org.junit.Test;
+
+public class ReverseLexicoderTest extends LexicoderTest {
+ public void testSortOrder() {
+ Comparator<Long> comp = Collections.reverseOrder();
+ assertSortOrder(new ReverseLexicoder<Long>(new LongLexicoder()), comp, Long.MIN_VALUE, 0xff1234567890abcdl, 0xffff1234567890abl, 0xffffff567890abcdl,
+ 0xffffffff7890abcdl, 0xffffffffff90abcdl, 0xffffffffffffabcdl, 0xffffffffffffffcdl, -1l, 0l, 0x01l, 0x1234l, 0x123456l, 0x12345678l, 0x1234567890l,
+ 0x1234567890abl, 0x1234567890abcdl, 0x1234567890abcdefl, Long.MAX_VALUE);
+
+ Comparator<String> comp2 = Collections.reverseOrder();
+ assertSortOrder(new ReverseLexicoder<String>(new StringLexicoder()), comp2, "a", "aa", "ab", "b", "aab");
+
+ }
+
+ /**
+ * Just a simple test verifying reverse indexed dates
- *
- * @throws UnsupportedEncodingException
+ */
+ @Test
+ public void testReverseSortDates() throws UnsupportedEncodingException {
+
+ ReverseLexicoder<Date> revLex = new ReverseLexicoder<Date>(new DateLexicoder());
+
+ Date date1 = new Date();
+ Date date2 = new Date(System.currentTimeMillis() + 10000);
+ Date date3 = new Date(System.currentTimeMillis() + 500);
+
+ Comparator<Date> comparator = Collections.reverseOrder();
+ assertSortOrder(revLex, comparator, date1, date2, date3);
+
+ // truncate date to hours
+ long time = System.currentTimeMillis() - (System.currentTimeMillis() % 3600000);
+ Date date = new Date(time);
+
+ System.out.println(date);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
index e83453e,3ec9bb1..13490e0
--- a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
@@@ -57,23 -55,9 +57,21 @@@ public class AccumuloInputFormatTest
private static final String PREFIX = AccumuloInputFormatTest.class.getSimpleName();
private static final String INSTANCE_NAME = PREFIX + "_mapred_instance";
private static final String TEST_TABLE_1 = PREFIX + "_mapred_table_1";
-
+
+ private JobConf job;
+
+ @BeforeClass
+ public static void setupClass() {
+ System.setProperty("hadoop.tmp.dir", System.getProperty("user.dir") + "/target/hadoop-tmp");
+ }
+
+ @Before
+ public void createJob() {
+ job = new JobConf();
+ }
+
/**
* Check that the iterator configuration is getting stored in the Job conf correctly.
- *
- * @throws IOException
*/
@Test
public void testSetIterator() throws IOException {
@@@ -152,11 -141,9 +150,9 @@@
assertEquals(list.get(1).getOptions().get(key), value);
assertEquals(list.get(1).getOptions().get(key + "2"), value);
}
-
+
/**
* Test getting iterator settings for multiple iterators set
- *
- * @throws IOException
*/
@Test
public void testGetIteratorSettings() throws IOException {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
index 54bd127,ae5e395..2500972
--- a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
@@@ -64,9 -64,45 +64,7 @@@ public class AccumuloInputFormatTest
private static final String PREFIX = AccumuloInputFormatTest.class.getSimpleName();
/**
- * Test basic setting & getting of max versions.
- *
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- @Deprecated
- @Test
- public void testMaxVersions() throws IOException {
- Job job = new Job();
- AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 1);
- int version = AccumuloInputFormat.getMaxVersions(job.getConfiguration());
- assertEquals(1, version);
- }
-
- /**
- * Test max versions with an invalid value.
- *
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- @Deprecated
- @Test(expected = IOException.class)
- public void testMaxVersionsLessThan1() throws IOException {
- Job job = new Job();
- AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 0);
- }
-
- /**
- * Test no max version configured.
- */
- @Deprecated
- @Test
- public void testNoMaxVersion() throws IOException {
- Job job = new Job();
- assertEquals(-1, AccumuloInputFormat.getMaxVersions(job.getConfiguration()));
- }
-
- /**
* Check that the iterator configuration is getting stored in the Job conf correctly.
- *
- * @throws IOException
*/
@Test
public void testSetIterator() throws IOException {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
index 009be17,0000000..c06df51
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
@@@ -1,317 -1,0 +1,309 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.core.client.mock;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Random;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.NamespaceNotEmptyException;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.NamespaceOperations;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.Filter;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.security.Authorizations;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class MockNamespacesTest {
+
+ Random random = new Random();
+ public static TemporaryFolder folder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
+
+ /**
+ * This test creates a table without specifying a namespace. In this case, it puts the table into the default namespace.
- *
- * @throws Exception
+ */
+ @Test
+ public void testDefaultNamespace() throws Exception {
+ String tableName = "test";
+ Instance instance = new MockInstance("default");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ assertTrue(c.namespaceOperations().exists(Namespaces.DEFAULT_NAMESPACE));
+ c.tableOperations().create(tableName);
+ assertTrue(c.tableOperations().exists(tableName));
+ }
+
+ /**
+ * This test creates a new namespace "testing" and a table "testing.table1" which puts "table1" into the "testing" namespace. Then we create "testing.table2"
+ * which creates "table2" and puts it into "testing" as well. Then we make sure that you can't delete a namespace with tables in it, and then we delete the
+ * tables and delete the namespace.
- *
- * @throws Exception
+ */
+ @Test
+ public void testCreateAndDeleteNamespace() throws Exception {
+ String namespace = "testing";
+ String tableName1 = namespace + ".table1";
+ String tableName2 = namespace + ".table2";
+
+ Instance instance = new MockInstance("createdelete");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ c.namespaceOperations().create(namespace);
+ assertTrue(c.namespaceOperations().exists(namespace));
+
+ c.tableOperations().create(tableName1);
+ assertTrue(c.tableOperations().exists(tableName1));
+
+ c.tableOperations().create(tableName2);
+ assertTrue(c.tableOperations().exists(tableName2));
+
+ // deleting
+ try {
+ // can't delete a namespace with tables in it
+ c.namespaceOperations().delete(namespace);
+ fail();
+ } catch (NamespaceNotEmptyException e) {
+ // ignore, supposed to happen
+ }
+ assertTrue(c.namespaceOperations().exists(namespace));
+ assertTrue(c.tableOperations().exists(tableName1));
+ assertTrue(c.tableOperations().exists(tableName2));
+
+ c.tableOperations().delete(tableName2);
+ assertTrue(!c.tableOperations().exists(tableName2));
+ assertTrue(c.namespaceOperations().exists(namespace));
+
+ c.tableOperations().delete(tableName1);
+ assertTrue(!c.tableOperations().exists(tableName1));
+ c.namespaceOperations().delete(namespace);
+ assertTrue(!c.namespaceOperations().exists(namespace));
+ }
+
+ /**
+ * This test creates a namespace, modifies it's properties, and checks to make sure that those properties are applied to its tables. To do something on a
+ * namespace-wide level, use {@link NamespaceOperations}.
+ *
+ * Checks to make sure namespace-level properties are overridden by table-level properties.
+ *
+ * Checks to see if the default namespace's properties work as well.
- *
- * @throws Exception
+ */
+
+ @Test
+ public void testNamespaceProperties() throws Exception {
+ String namespace = "propchange";
+ String tableName1 = namespace + ".table1";
+ String tableName2 = namespace + ".table2";
+
+ String propKey = Property.TABLE_SCAN_MAXMEM.getKey();
+ String propVal = "42K";
+
+ Instance instance = new MockInstance("props");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(tableName1);
+ c.namespaceOperations().setProperty(namespace, propKey, propVal);
+
+ // check the namespace has the property
+ assertTrue(checkNamespaceHasProp(c, namespace, propKey, propVal));
+
+ // check that the table gets it from the namespace
+ assertTrue(checkTableHasProp(c, tableName1, propKey, propVal));
+
+ // test a second table to be sure the first wasn't magical
+ // (also, changed the order, the namespace has the property already)
+ c.tableOperations().create(tableName2);
+ assertTrue(checkTableHasProp(c, tableName2, propKey, propVal));
+
+ // test that table properties override namespace properties
+ String propKey2 = Property.TABLE_FILE_MAX.getKey();
+ String propVal2 = "42";
+ String tablePropVal = "13";
+
+ c.tableOperations().setProperty(tableName2, propKey2, tablePropVal);
+ c.namespaceOperations().setProperty("propchange", propKey2, propVal2);
+
+ assertTrue(checkTableHasProp(c, tableName2, propKey2, tablePropVal));
+
+ // now check that you can change the default namespace's properties
+ propVal = "13K";
+ String tableName = "some_table";
+ c.tableOperations().create(tableName);
+ c.namespaceOperations().setProperty(Namespaces.DEFAULT_NAMESPACE, propKey, propVal);
+
+ assertTrue(checkTableHasProp(c, tableName, propKey, propVal));
+
+ // test the properties server-side by configuring an iterator.
+ // should not show anything with column-family = 'a'
+ String tableName3 = namespace + ".table3";
+ c.tableOperations().create(tableName3);
+
+ IteratorSetting setting = new IteratorSetting(250, "thing", SimpleFilter.class.getName());
+ c.namespaceOperations().attachIterator(namespace, setting);
+
+ BatchWriter bw = c.createBatchWriter(tableName3, new BatchWriterConfig());
+ Mutation m = new Mutation("r");
+ m.put("a", "b", new Value("abcde".getBytes()));
+ bw.addMutation(m);
+ bw.flush();
+ bw.close();
+
+ // Scanner s = c.createScanner(tableName3, Authorizations.EMPTY);
+ // do scanners work correctly in mock?
+ // assertTrue(!s.iterator().hasNext());
+ }
+
+ /**
+ * This test renames and clones two separate table into different namespaces. different namespace.
- *
- * @throws Exception
+ */
+ @Test
+ public void testRenameAndCloneTableToNewNamespace() throws Exception {
+ String namespace1 = "renamed";
+ String namespace2 = "cloned";
+ String tableName = "table";
+ String tableName1 = "renamed.table1";
+ // String tableName2 = "cloned.table2";
+
+ Instance instance = new MockInstance("renameclone");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ c.tableOperations().create(tableName);
+ c.namespaceOperations().create(namespace1);
+ c.namespaceOperations().create(namespace2);
+
+ c.tableOperations().rename(tableName, tableName1);
+
+ assertTrue(c.tableOperations().exists(tableName1));
+ assertTrue(!c.tableOperations().exists(tableName));
+
+ // TODO implement clone in mock
+ /*
+ * c.tableOperations().clone(tableName1, tableName2, false, null, null);
+ *
+ * assertTrue(c.tableOperations().exists(tableName1)); assertTrue(c.tableOperations().exists(tableName2));
+ */
+ return;
+ }
+
+ /**
+ * This test renames a namespace and ensures that its tables are still correct
+ */
+ @Test
+ public void testNamespaceRename() throws Exception {
+ String namespace1 = "n1";
+ String namespace2 = "n2";
+ String table = "t";
+
+ Instance instance = new MockInstance("rename");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ c.namespaceOperations().create(namespace1);
+ c.tableOperations().create(namespace1 + "." + table);
+
+ c.namespaceOperations().rename(namespace1, namespace2);
+
+ assertTrue(!c.namespaceOperations().exists(namespace1));
+ assertTrue(c.namespaceOperations().exists(namespace2));
+ assertTrue(!c.tableOperations().exists(namespace1 + "." + table));
+ assertTrue(c.tableOperations().exists(namespace2 + "." + table));
+ }
+
+ /**
+ * This tests adding iterators to a namespace, listing them, and removing them
+ */
+ @Test
+ public void testNamespaceIterators() throws Exception {
+ Instance instance = new MockInstance("Iterators");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ String namespace = "iterator";
+ String tableName = namespace + ".table";
+ String iter = "thing";
+
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(tableName);
+
+ IteratorSetting setting = new IteratorSetting(250, iter, SimpleFilter.class.getName());
+ HashSet<IteratorScope> scope = new HashSet<IteratorScope>();
+ scope.add(IteratorScope.scan);
+ c.namespaceOperations().attachIterator(namespace, setting, EnumSet.copyOf(scope));
+
+ BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
+ Mutation m = new Mutation("r");
+ m.put("a", "b", new Value("abcde".getBytes(Constants.UTF8)));
+ bw.addMutation(m);
+ bw.flush();
+
+ Scanner s = c.createScanner(tableName, Authorizations.EMPTY);
+ System.out.println(s.iterator().next());
+ // do scanners work correctly in mock?
+ // assertTrue(!s.iterator().hasNext());
+
+ assertTrue(c.namespaceOperations().listIterators(namespace).containsKey(iter));
+ c.namespaceOperations().removeIterator(namespace, iter, EnumSet.copyOf(scope));
+ }
+
+ private boolean checkTableHasProp(Connector c, String t, String propKey, String propVal) throws AccumuloException, TableNotFoundException {
+ for (Entry<String,String> e : c.tableOperations().getProperties(t)) {
+ if (e.getKey().equals(propKey) && e.getValue().equals(propVal)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean checkNamespaceHasProp(Connector c, String n, String propKey, String propVal) throws AccumuloException, NamespaceNotFoundException,
+ AccumuloSecurityException {
+ for (Entry<String,String> e : c.namespaceOperations().getProperties(n)) {
+ if (e.getKey().equals(propKey) && e.getValue().equals(propVal)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public static class SimpleFilter extends Filter {
+ @Override
+ public boolean accept(Key k, Value v) {
+ if (k.getColumnFamily().toString().equals("a"))
+ return false;
+ return true;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
index de6ca21,0000000..d31a788
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
@@@ -1,106 -1,0 +1,103 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.security;
+
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.constraints.Constraint.Environment;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.Mutation;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class VisibilityConstraintTest {
+
+ VisibilityConstraint vc;
+ Environment env;
+ Mutation mutation;
+
+ static final ColumnVisibility good = new ColumnVisibility("good");
+ static final ColumnVisibility bad = new ColumnVisibility("bad");
+
+ static final String D = "don't care";
+
+ static final List<Short> ENOAUTH = Arrays.asList((short) 2);
+
- /**
- * @throws java.lang.Exception
- */
+ @Before
+ public void setUp() throws Exception {
+ vc = new VisibilityConstraint();
+ mutation = new Mutation("r");
+
+ ArrayByteSequence bs = new ArrayByteSequence("good".getBytes(Constants.UTF8));
+
+ AuthorizationContainer ac = createNiceMock(AuthorizationContainer.class);
+ expect(ac.contains(bs)).andReturn(true);
+ replay(ac);
+
+ env = createMock(Environment.class);
+ expect(env.getAuthorizationsContainer()).andReturn(ac);
+ replay(env);
+ }
+
+ @Test
+ public void testNoVisibility() {
+ mutation.put(D, D, D);
+ assertNull("authorized", vc.check(env, mutation));
+ }
+
+ @Test
+ public void testVisibilityNoAuth() {
+ mutation.put(D, D, bad, D);
+ assertEquals("unauthorized", ENOAUTH, vc.check(env, mutation));
+ }
+
+ @Test
+ public void testGoodVisibilityAuth() {
+ mutation.put(D, D, good, D);
+ assertNull("authorized", vc.check(env, mutation));
+ }
+
+ @Test
+ public void testCachedVisibilities() {
+ mutation.put(D, D, good, "v");
+ mutation.put(D, D, good, "v2");
+ assertNull("authorized", vc.check(env, mutation));
+ }
+
+ @Test
+ public void testMixedVisibilities() {
+ mutation.put(D, D, bad, D);
+ mutation.put(D, D, good, D);
+ assertEquals("unauthorized", ENOAUTH, vc.check(env, mutation));
+ }
+
+ @Test
+ @Ignore
+ public void testMalformedVisibility() {
+ // TODO: ACCUMULO-1006 Should test for returning error code 1, but not sure how since ColumnVisibility won't let us construct a bad one in the first place
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
index 4607fdb,e76352a..44947d1
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
@@@ -89,40 -89,29 +89,36 @@@ public class RandomBatchWriter
// create a random value that is a function of the
// row id for verification purposes
byte value[] = createValue(rowid, dataSize);
-
+
m.put(new Text("foo"), new Text("1"), visibility, new Value(value));
-
+
return m;
}
-
+
static class Opts extends ClientOnRequiredTable {
- @Parameter(names="--num", required=true)
+ @Parameter(names = "--num", required = true)
int num = 0;
- @Parameter(names="--min")
+ @Parameter(names = "--min")
long min = 0;
- @Parameter(names="--max")
+ @Parameter(names = "--max")
long max = Long.MAX_VALUE;
- @Parameter(names="--size", required=true, description="size of the value to write")
+ @Parameter(names = "--size", required = true, description = "size of the value to write")
int size = 0;
- @Parameter(names="--vis", converter=VisibilityConverter.class)
+ @Parameter(names = "--vis", converter = VisibilityConverter.class)
ColumnVisibility visiblity = new ColumnVisibility("");
- @Parameter(names="--seed", description="seed for pseudo-random number generator")
+ @Parameter(names = "--seed", description = "seed for pseudo-random number generator")
Long seed = null;
}
-
+
+ public static long abs(long l) {
+ l = Math.abs(l); // abs(Long.MIN_VALUE) == Long.MIN_VALUE...
+ if (l < 0)
+ return 0;
+ return l;
+ }
+
/**
* Writes a specified number of entries to Accumulo using a {@link BatchWriter}.
- *
- * @throws AccumuloException
- * @throws AccumuloSecurityException
- * @throws TableNotFoundException
*/
public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
Opts opts = new Opts();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/NGramIngest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
index 3a211e2,669c76d..30ebd06
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
@@@ -121,9 -120,10 +121,8 @@@ public class TableToFile extends Config
*
* @param args
* instanceName zookeepers username password table columns outputpath
- * @throws Exception
*/
public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(CachedConfiguration.getInstance(), new TableToFile(), args);
- if (res != 0)
- System.exit(res);
+ ToolRunner.run(new Configuration(), new TableToFile(), args);
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
index add4c4c,d98d78b..aa12c71
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
@@@ -59,26 -66,10 +59,23 @@@ public class Query
IntersectingIterator.setColumnFamilies(ii, columns);
bs.addScanIterator(ii);
bs.setRanges(Collections.singleton(new Range()));
+ List<String> result = new ArrayList<String>();
for (Entry<Key,Value> entry : bs) {
- System.out.println(" " + entry.getKey().getColumnQualifier());
+ result.add(entry.getKey().getColumnQualifier().toString());
}
-
+ return result;
+ }
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ BatchScannerOpts bsOpts = new BatchScannerOpts();
+ opts.parseArgs(Query.class.getName(), args, bsOpts);
+ Connector conn = opts.getConnector();
+ BatchScanner bs = conn.createBatchScanner(opts.tableName, opts.auths, bsOpts.scanThreads);
+ bs.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);
+
+ for (String entry : query(bs, opts.terms))
+ System.out.println(" " + entry);
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java
----------------------------------------------------------------------
diff --cc minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java
index f7070dc,0000000..ab84d37
mode 100644,000000..100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java
@@@ -1,264 -1,0 +1,262 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.minicluster;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.ServerSocket;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.regex.Pattern;
+
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.commons.io.FileUtils;
+
+import com.beust.jcommander.IStringConverter;
+import com.beust.jcommander.Parameter;
+import com.google.common.io.Files;
+
+/**
+ * A runner for starting up a {@link MiniAccumuloCluster} from the command line using an optional configuration properties file. An example property file looks
+ * like the following:
+ *
+ * <pre>
+ * rootPassword=secret
+ * instanceName=testInstance
+ * numTServers=1
+ * zooKeeperPort=3191
+ * jdwpEnabled=true
+ * zooKeeperMemory=128M
+ * tserverMemory=256M
+ * masterMemory=128M
+ * defaultMemory=256M
+ * shutdownPort=4446
+ * site.instance.secret=HUSH
+ * </pre>
+ *
+ * All items in the properties file above are optional and a default value will be provided in their absence. Any site configuration properties (typically found
+ * in the accumulo-site.xml file) should be prefixed with "site." in the properties file.
+ *
+ * @since 1.6.0
+ */
+public class MiniAccumuloRunner {
+ private static final String ROOT_PASSWORD_PROP = "rootPassword";
+ private static final String SHUTDOWN_PORT_PROP = "shutdownPort";
+ private static final String DEFAULT_MEMORY_PROP = "defaultMemory";
+ private static final String MASTER_MEMORY_PROP = "masterMemory";
+ private static final String TSERVER_MEMORY_PROP = "tserverMemory";
+ private static final String ZOO_KEEPER_MEMORY_PROP = "zooKeeperMemory";
+ private static final String JDWP_ENABLED_PROP = "jdwpEnabled";
+ private static final String ZOO_KEEPER_PORT_PROP = "zooKeeperPort";
+ private static final String NUM_T_SERVERS_PROP = "numTServers";
+ private static final String DIRECTORY_PROP = "directory";
+ private static final String INSTANCE_NAME_PROP = "instanceName";
+
+ private static void printProperties() {
+ System.out.println("#mini Accumulo cluster runner properties.");
+ System.out.println("#");
+ System.out.println("#uncomment following propeties to use, propeties not set will use default or random value");
+ System.out.println();
+ System.out.println("#" + INSTANCE_NAME_PROP + "=devTest");
+ System.out.println("#" + DIRECTORY_PROP + "=/tmp/mac1");
+ System.out.println("#" + ROOT_PASSWORD_PROP + "=secret");
+ System.out.println("#" + NUM_T_SERVERS_PROP + "=2");
+ System.out.println("#" + ZOO_KEEPER_PORT_PROP + "=40404");
+ System.out.println("#" + SHUTDOWN_PORT_PROP + "=41414");
+ System.out.println("#" + DEFAULT_MEMORY_PROP + "=128M");
+ System.out.println("#" + MASTER_MEMORY_PROP + "=128M");
+ System.out.println("#" + TSERVER_MEMORY_PROP + "=128M");
+ System.out.println("#" + ZOO_KEEPER_MEMORY_PROP + "=128M");
+ System.out.println("#" + JDWP_ENABLED_PROP + "=false");
+
+ System.out.println();
+ System.out.println("# Configuration normally placed in accumulo-site.xml can be added using a site. prefix.");
+ System.out.println("# For example the following line will set tserver.compaction.major.concurrent.max");
+ System.out.println();
+ System.out.println("#site.tserver.compaction.major.concurrent.max=4");
+
+ }
+
+ public static class PropertiesConverter implements IStringConverter<Properties> {
+ @Override
+ public Properties convert(String fileName) {
+ Properties prop = new Properties();
+ InputStream is;
+ try {
+ is = new FileInputStream(fileName);
+ try {
+ prop.load(is);
+ } finally {
+ is.close();
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return prop;
+ }
+ }
+
+ private static final String FORMAT_STRING = " %-21s %s";
+
+ public static class Opts extends Help {
+ @Parameter(names = "-p", required = false, description = "properties file name", converter = PropertiesConverter.class)
+ Properties prop = new Properties();
+
+ @Parameter(names = {"-c", "--printProperties"}, required = false, description = "prints an example propeties file, redirect to file to use")
+ boolean printProps = false;
+ }
+
+ /**
+ * Runs the {@link MiniAccumuloCluster} given a -p argument with a property file. Establishes a shutdown port for asynchronous operation.
+ *
+ * @param args
+ * An optional -p argument can be specified with the path to a valid properties file.
- *
- * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(MiniAccumuloRunner.class.getName(), args);
+
+ if (opts.printProps) {
+ printProperties();
+ System.exit(0);
+ }
+
+ int shutdownPort = 4445;
+
+ final File miniDir;
+
+ if (opts.prop.containsKey(DIRECTORY_PROP))
+ miniDir = new File(opts.prop.getProperty(DIRECTORY_PROP));
+ else
+ miniDir = Files.createTempDir();
+
+ String rootPass = opts.prop.containsKey(ROOT_PASSWORD_PROP) ? opts.prop.getProperty(ROOT_PASSWORD_PROP) : "secret";
+
+ MiniAccumuloConfig config = new MiniAccumuloConfig(miniDir, rootPass);
+
+ if (opts.prop.containsKey(INSTANCE_NAME_PROP))
+ config.setInstanceName(opts.prop.getProperty(INSTANCE_NAME_PROP));
+ if (opts.prop.containsKey(NUM_T_SERVERS_PROP))
+ config.setNumTservers(Integer.parseInt(opts.prop.getProperty(NUM_T_SERVERS_PROP)));
+ if (opts.prop.containsKey(ZOO_KEEPER_PORT_PROP))
+ config.setZooKeeperPort(Integer.parseInt(opts.prop.getProperty(ZOO_KEEPER_PORT_PROP)));
+ if (opts.prop.containsKey(JDWP_ENABLED_PROP))
+ config.setJDWPEnabled(Boolean.parseBoolean(opts.prop.getProperty(JDWP_ENABLED_PROP)));
+ if (opts.prop.containsKey(ZOO_KEEPER_MEMORY_PROP))
+ setMemoryOnConfig(config, opts.prop.getProperty(ZOO_KEEPER_MEMORY_PROP), ServerType.ZOOKEEPER);
+ if (opts.prop.containsKey(TSERVER_MEMORY_PROP))
+ setMemoryOnConfig(config, opts.prop.getProperty(TSERVER_MEMORY_PROP), ServerType.TABLET_SERVER);
+ if (opts.prop.containsKey(MASTER_MEMORY_PROP))
+ setMemoryOnConfig(config, opts.prop.getProperty(MASTER_MEMORY_PROP), ServerType.MASTER);
+ if (opts.prop.containsKey(DEFAULT_MEMORY_PROP))
+ setMemoryOnConfig(config, opts.prop.getProperty(DEFAULT_MEMORY_PROP));
+ if (opts.prop.containsKey(SHUTDOWN_PORT_PROP))
+ shutdownPort = Integer.parseInt(opts.prop.getProperty(SHUTDOWN_PORT_PROP));
+
+ Map<String,String> siteConfig = new HashMap<String,String>();
+ for (Map.Entry<Object,Object> entry : opts.prop.entrySet()) {
+ String key = (String) entry.getKey();
+ if (key.startsWith("site."))
+ siteConfig.put(key.replaceFirst("site.", ""), (String) entry.getValue());
+ }
+
+ config.setSiteConfig(siteConfig);
+
+ final MiniAccumuloCluster accumulo = new MiniAccumuloCluster(config);
+
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ try {
+ accumulo.stop();
+ FileUtils.deleteDirectory(miniDir);
+ System.out.println("\nShut down gracefully on " + new Date());
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+
+ accumulo.start();
+
+ printInfo(accumulo, shutdownPort);
+
+ // start a socket on the shutdown port and block- anything connected to this port will activate the shutdown
+ ServerSocket shutdownServer = new ServerSocket(shutdownPort);
+ shutdownServer.accept();
+
+ System.exit(0);
+ }
+
+ private static boolean validateMemoryString(String memoryString) {
+ String unitsRegex = "[";
+ MemoryUnit[] units = MemoryUnit.values();
+ for (int i = 0; i < units.length; i++) {
+ unitsRegex += units[i].suffix();
+ if (i < units.length - 1)
+ unitsRegex += "|";
+ }
+ unitsRegex += "]";
+ Pattern p = Pattern.compile("\\d+" + unitsRegex);
+ return p.matcher(memoryString).matches();
+ }
+
+ private static void setMemoryOnConfig(MiniAccumuloConfig config, String memoryString) {
+ setMemoryOnConfig(config, memoryString, null);
+ }
+
+ private static void setMemoryOnConfig(MiniAccumuloConfig config, String memoryString, ServerType serverType) {
+ if (!validateMemoryString(memoryString))
+ throw new IllegalArgumentException(memoryString + " is not a valid memory string");
+
+ long memSize = Long.parseLong(memoryString.substring(0, memoryString.length() - 1));
+ MemoryUnit memUnit = MemoryUnit.fromSuffix(memoryString.substring(memoryString.length() - 1));
+
+ if (serverType != null)
+ config.setMemory(serverType, memSize, memUnit);
+ else
+ config.setDefaultMemory(memSize, memUnit);
+ }
+
+ private static void printInfo(MiniAccumuloCluster accumulo, int shutdownPort) {
+ System.out.println("Mini Accumulo Cluster\n");
+ System.out.println(String.format(FORMAT_STRING, "Directory:", accumulo.getConfig().getDir().getAbsoluteFile()));
+ System.out.println(String.format(FORMAT_STRING, "Logs:", accumulo.getConfig().getImpl().getLogDir().getAbsoluteFile()));
+ System.out.println(String.format(FORMAT_STRING, "Instance Name:", accumulo.getConfig().getInstanceName()));
+ System.out.println(String.format(FORMAT_STRING, "Root Password:", accumulo.getConfig().getRootPassword()));
+ System.out.println(String.format(FORMAT_STRING, "ZooKeeper:", accumulo.getZooKeepers()));
+
+ for (Pair<ServerType,Integer> pair : accumulo.getDebugPorts()) {
+ System.out.println(String.format(FORMAT_STRING, pair.getFirst().prettyPrint() + " JDWP Host:", "localhost:" + pair.getSecond()));
+ }
+
+ System.out.println(String.format(FORMAT_STRING, "Shutdown Port:", shutdownPort));
+
+ System.out.println();
+ System.out.println(" To connect with shell, use the following command : ");
+ System.out.println(" accumulo shell -zh " + accumulo.getZooKeepers() + " -zi " + accumulo.getConfig().getInstanceName() + " -u root ");
+
+ System.out.println("\n\nSuccessfully started on " + new Date());
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java
index 442294f,0000000..05806ca
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java
@@@ -1,30 -1,0 +1,27 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.conf;
+
+import org.apache.accumulo.server.client.HdfsZooInstance;
+
+public class ConfigSanityCheck {
+
- /**
- * @param args
- */
+ public static void main(String[] args) {
+ new ServerConfiguration(HdfsZooInstance.getInstance()).getConfiguration();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
index fd76ce2,0000000..5bd1632
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
@@@ -1,151 -1,0 +1,149 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.balancer;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletMigration;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransportException;
+
+public abstract class TabletBalancer {
+
+ private static final Logger log = Logger.getLogger(TabletBalancer.class);
+
+ protected ServerConfiguration configuration;
+
+ /**
+ * Initialize the TabletBalancer. This gives the balancer the opportunity to read the configuration.
+ */
+ public void init(ServerConfiguration conf) {
+ configuration = conf;
+ }
+
+ /**
+ * Assign tablets to tablet servers. This method is called whenever the master finds tablets that are unassigned.
+ *
+ * @param current
+ * The current table-summary state of all the online tablet servers. Read-only. The TabletServerStatus for each server may be null if the tablet
+ * server has not yet responded to a recent request for status.
+ * @param unassigned
+ * A map from unassigned tablet to the last known tablet server. Read-only.
+ * @param assignments
+ * A map from tablet to assigned server. Write-only.
+ */
+ abstract public void getAssignments(SortedMap<TServerInstance,TabletServerStatus> current, Map<KeyExtent,TServerInstance> unassigned,
+ Map<KeyExtent,TServerInstance> assignments);
+
+ /**
+ * Ask the balancer if any migrations are necessary.
+ *
+ * @param current
+ * The current table-summary state of all the online tablet servers. Read-only.
+ * @param migrations
+ * the current set of migrations. Read-only.
+ * @param migrationsOut
+ * new migrations to perform; should not contain tablets in the current set of migrations. Write-only.
+ * @return the time, in milliseconds, to wait before re-balancing.
+ *
+ * This method will not be called when there are unassigned tablets.
+ */
+ public abstract long balance(SortedMap<TServerInstance,TabletServerStatus> current, Set<KeyExtent> migrations, List<TabletMigration> migrationsOut);
+
+ /**
+ * Fetch the tablets for the given table by asking the tablet server. Useful if your balance strategy needs details at the tablet level to decide what tablets
+ * to move.
+ *
+ * @param tserver
+ * The tablet server to ask.
+ * @param tableId
+ * The table id
+ * @return a list of tablet statistics
+ * @throws ThriftSecurityException
+ * tablet server disapproves of your internal System password.
+ * @throws TException
+ * any other problem
+ */
+ public List<TabletStats> getOnlineTabletsForTable(TServerInstance tserver, String tableId) throws ThriftSecurityException, TException {
+ log.debug("Scanning tablet server " + tserver + " for table " + tableId);
+ Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), tserver.getLocation(), configuration.getConfiguration());
+ try {
+ List<TabletStats> onlineTabletsForTable = client.getTabletStats(Tracer.traceInfo(), SystemCredentials.get().toThrift(configuration.getInstance()),
+ tableId);
+ return onlineTabletsForTable;
+ } catch (TTransportException e) {
+ log.error("Unable to connect to " + tserver + ": " + e);
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ return null;
+ }
+
+ /**
+ * Utility to ensure that the migrations from balance() are consistent:
+ * <ul>
+ * <li>Tablet objects are not null
+ * <li>Source and destination tablet servers are not null and current
+ * </ul>
+ *
- * @param current
- * @param migrations
+ * @return A list of TabletMigration object that passed sanity checks.
+ */
+ public static List<TabletMigration> checkMigrationSanity(Set<TServerInstance> current, List<TabletMigration> migrations) {
+ List<TabletMigration> result = new ArrayList<TabletMigration>(migrations.size());
+ for (TabletMigration m : migrations) {
+ if (m.tablet == null) {
+ log.warn("Balancer gave back a null tablet " + m);
+ continue;
+ }
+ if (m.newServer == null) {
+ log.warn("Balancer did not set the destination " + m);
+ continue;
+ }
+ if (m.oldServer == null) {
+ log.warn("Balancer did not set the source " + m);
+ continue;
+ }
+ if (!current.contains(m.oldServer)) {
+ log.warn("Balancer wants to move a tablet from a server that is not current: " + m);
+ continue;
+ }
+ if (!current.contains(m.newServer)) {
+ log.warn("Balancer wants to move a tablet to a server that is not current: " + m);
+ continue;
+ }
+ result.add(m);
+ }
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
index e898109,0000000..7c75454
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
@@@ -1,91 -1,0 +1,84 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.state;
+
+import java.util.Collection;
+import java.util.Collections;
+
+/**
+ * Interface for storing information about tablet assignments. There are three implementations:
+ *
+ * ZooTabletStateStore: information about the root tablet is stored in ZooKeeper MetaDataStateStore: information about the other tablets are stored in the
+ * metadata table
+ *
+ */
+public abstract class TabletStateStore implements Iterable<TabletLocationState> {
+
+ /**
+ * Identifying name for this tablet state store.
+ */
+ abstract public String name();
+
+ /**
+ * Scan the information about the tablets covered by this store
+ */
+ @Override
+ abstract public ClosableIterator<TabletLocationState> iterator();
+
+ /**
+ * Store the assigned locations in the data store.
- *
- * @param assignments
- * @throws DistributedStoreException
+ */
+ abstract public void setFutureLocations(Collection<Assignment> assignments) throws DistributedStoreException;
+
+ /**
+ * Tablet servers will update the data store with the location when they bring the tablet online
- *
- * @param assignments
- * @throws DistributedStoreException
+ */
+ abstract public void setLocations(Collection<Assignment> assignments) throws DistributedStoreException;
+
+ /**
+ * Mark the tablets as having no known or future location.
+ *
+ * @param tablets
+ * the tablets' current information
- * @throws DistributedStoreException
+ */
+ abstract public void unassign(Collection<TabletLocationState> tablets) throws DistributedStoreException;
+
+ public static void unassign(TabletLocationState tls) throws DistributedStoreException {
+ TabletStateStore store;
+ if (tls.extent.isRootTablet()) {
+ store = new ZooTabletStateStore();
+ } else if (tls.extent.isMeta()) {
+ store = new RootTabletStateStore();
+ } else {
+ store = new MetaDataStateStore();
+ }
+ store.unassign(Collections.singletonList(tls));
+ }
+
+ public static void setLocation(Assignment assignment) throws DistributedStoreException {
+ TabletStateStore store;
+ if (assignment.tablet.isRootTablet()) {
+ store = new ZooTabletStateStore();
+ } else if (assignment.tablet.isMeta()) {
+ store = new RootTabletStateStore();
+ } else {
+ store = new MetaDataStateStore();
+ }
+ store.setLocations(Collections.singletonList(assignment));
+ }
+
+}
[03/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
index d018228,0000000..f4a8082
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
@@@ -1,136 -1,0 +1,135 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+public class AddFilesWithMissingEntries {
+
+ static final Logger log = Logger.getLogger(AddFilesWithMissingEntries.class);
+
+ public static class Opts extends ClientOpts {
+ @Parameter(names="-update", description="Make changes to the !METADATA table to include missing files")
+ boolean update = false;
+ }
+
+
+ /**
+ * A utility to add files to the !METADATA table that are not listed in the root tablet.
+ * This is a recovery tool for someone who knows what they are doing. It might be better to
+ * save off files, and recover your instance by re-initializing and importing the existing files.
+ *
- * @param args
+ */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(AddFilesWithMissingEntries.class.getName(), args, bwOpts);
+
+ final Key rootTableEnd = new Key(Constants.ROOT_TABLET_EXTENT.getEndRow());
+ final Range range = new Range(rootTableEnd.followingKey(PartialKey.ROW), true, Constants.METADATA_RESERVED_KEYSPACE_START_KEY, false);
+ final Scanner scanner = opts.getConnector().createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ scanner.setRange(range);
+ final Configuration conf = new Configuration();
+ final FileSystem fs = FileSystem.get(conf);
+
+ KeyExtent last = new KeyExtent();
+ String directory = null;
+ Set<String> knownFiles = new HashSet<String>();
+
+ int count = 0;
+ final MultiTableBatchWriter writer = opts.getConnector().createMultiTableBatchWriter(bwOpts.getBatchWriterConfig());
+
+ // collect the list of known files and the directory for each extent
+ for (Entry<Key,Value> entry : scanner) {
+ Key key = entry.getKey();
+ KeyExtent ke = new KeyExtent(key.getRow(), (Text) null);
+ // when the key extent changes
+ if (!ke.equals(last)) {
+ if (directory != null) {
+ // add any files in the directory unknown to the key extent
+ count += addUnknownFiles(fs, directory, knownFiles, last, writer, opts.update);
+ }
+ directory = null;
+ knownFiles.clear();
+ last = ke;
+ }
+ if (Constants.METADATA_DIRECTORY_COLUMN.hasColumns(key)) {
+ directory = entry.getValue().toString();
+ log.debug("Found directory " + directory + " for row " + key.getRow().toString());
+ } else if (key.compareColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY) == 0) {
+ String filename = key.getColumnQualifier().toString();
+ knownFiles.add(filename);
+ log.debug("METADATA file found: " + filename);
+ }
+ }
+ if (directory != null) {
+ // catch the last key extent
+ count += addUnknownFiles(fs, directory, knownFiles, last, writer, opts.update);
+ }
+ log.info("There were " + count + " files that are unknown to the metadata table");
+ writer.close();
+ }
+
+ private static int addUnknownFiles(FileSystem fs, String directory, Set<String> knownFiles, KeyExtent ke, MultiTableBatchWriter writer, boolean update) throws Exception {
+ int count = 0;
+ final String tableId = ke.getTableId().toString();
+ final Text row = ke.getMetadataEntry();
+ log.info(row.toString());
+ final Path path = new Path(ServerConstants.getTablesDir() + "/" + tableId + directory);
+ for (FileStatus file : fs.listStatus(path)) {
+ if (file.getPath().getName().endsWith("_tmp") || file.getPath().getName().endsWith("_tmp.rf"))
+ continue;
+ final String filename = directory + "/" + file.getPath().getName();
+ if (!knownFiles.contains(filename)) {
+ count++;
+ final Mutation m = new Mutation(row);
+ String size = Long.toString(file.getLen());
+ String entries = "1"; // lie
+ String value = size + "," + entries;
+ m.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(filename), new Value(value.getBytes(Constants.UTF8)));
+ if (update) {
+ writer.getBatchWriter(Constants.METADATA_TABLE_NAME).addMutation(m);
+ }
+ }
+ }
+ return count;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
index 95f6a32,0000000..3342993
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
@@@ -1,123 -1,0 +1,120 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+
+import com.beust.jcommander.Parameter;
+
+public class DumpZookeeper {
+
+ static IZooReaderWriter zk = null;
+
+ private static final Logger log = Logger.getLogger(DumpZookeeper.class);
+
+ private static class Encoded {
+ public String encoding;
+ public String value;
+
+ Encoded(String e, String v) {
+ encoding = e;
+ value = v;
+ }
+ }
+
+ static class Opts extends Help {
+ @Parameter(names="--root", description="the root of the znode tree to dump")
+ String root = "/";
+ }
+
- /**
- * @param args
- */
+ public static void main(String[] args) {
+ Opts opts = new Opts();
+ opts.parseArgs(DumpZookeeper.class.getName(), args);
+
+ Logger.getRootLogger().setLevel(Level.WARN);
+ PrintStream out = System.out;
+ try {
+ zk = ZooReaderWriter.getInstance();
+
+ write(out, 0, "<dump root='%s'>", opts.root);
+ for (String child : zk.getChildren(opts.root, null))
+ if (!child.equals("zookeeper"))
+ dump(out, opts.root, child, 1);
+ write(out, 0, "</dump>");
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ }
+
+ private static void dump(PrintStream out, String root, String child, int indent) throws KeeperException, InterruptedException, UnsupportedEncodingException {
+ String path = root + "/" + child;
+ if (root.endsWith("/"))
+ path = root + child;
+ Stat stat = zk.getStatus(path);
+ if (stat == null)
+ return;
+ String type = "node";
+ if (stat.getEphemeralOwner() != 0) {
+ type = "ephemeral";
+ }
+ if (stat.getNumChildren() == 0) {
+ if (stat.getDataLength() == 0) {
+ write(out, indent, "<%s name='%s'/>", type, child);
+ } else {
+ Encoded value = value(path);
+ write(out, indent, "<%s name='%s' encoding='%s' value='%s'/>", type, child, value.encoding, value.value);
+ }
+ } else {
+ if (stat.getDataLength() == 0) {
+ write(out, indent, "<%s name='%s'>", type, child);
+ } else {
+ Encoded value = value(path);
+ write(out, indent, "<%s name='%s' encoding='%s' value='%s'>", type, child, value.encoding, value.value);
+ }
+ for (String c : zk.getChildren(path, null)) {
+ dump(out, path, c, indent + 1);
+ }
+ write(out, indent, "</node>");
+ }
+ }
+
+ private static Encoded value(String path) throws KeeperException, InterruptedException, UnsupportedEncodingException {
+ byte[] data = zk.getData(path, null);
+ for (int i = 0; i < data.length; i++) {
+ // does this look like simple ascii?
+ if (data[i] < ' ' || data[i] > '~')
+ return new Encoded("base64", new String(Base64.encodeBase64(data), "utf8"));
+ }
+ return new Encoded("utf8", new String(data, "utf8"));
+ }
+
+ private static void write(PrintStream out, int indent, String fmt, Object... args) {
+ for (int i = 0; i < indent; i++)
+ out.print(" ");
+ out.println(String.format(fmt, args));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
index 60b50da,0000000..42ebbe2
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
@@@ -1,74 -1,0 +1,71 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.util.Iterator;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.server.master.LiveTServerSet;
+import org.apache.accumulo.server.master.LiveTServerSet.Listener;
+import org.apache.accumulo.server.master.state.MetaDataTableScanner;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletLocationState;
+import org.apache.accumulo.server.master.state.TabletState;
+import org.apache.accumulo.server.master.state.tables.TableManager;
+import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.commons.collections.iterators.IteratorChain;
+import org.apache.log4j.Logger;
+
+public class FindOfflineTablets {
+ private static final Logger log = Logger.getLogger(FindOfflineTablets.class);
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ ClientOpts opts = new ClientOpts();
+ opts.parseArgs(FindOfflineTablets.class.getName(), args);
+ final AtomicBoolean scanning = new AtomicBoolean(false);
+ Instance instance = opts.getInstance();
+ MetaDataTableScanner rootScanner = new MetaDataTableScanner(instance, SecurityConstants.getSystemCredentials(), Constants.METADATA_ROOT_TABLET_KEYSPACE);
+ MetaDataTableScanner metaScanner = new MetaDataTableScanner(instance, SecurityConstants.getSystemCredentials(), Constants.NON_ROOT_METADATA_KEYSPACE);
+ @SuppressWarnings("unchecked")
+ Iterator<TabletLocationState> scanner = (Iterator<TabletLocationState>)new IteratorChain(rootScanner, metaScanner);
+ LiveTServerSet tservers = new LiveTServerSet(instance, DefaultConfiguration.getDefaultConfiguration(), new Listener() {
+ @Override
+ public void update(LiveTServerSet current, Set<TServerInstance> deleted, Set<TServerInstance> added) {
+ if (!deleted.isEmpty() && scanning.get())
+ log.warn("Tablet servers deleted while scanning: " + deleted);
+ if (!added.isEmpty() && scanning.get())
+ log.warn("Tablet servers added while scanning: " + added);
+ }
+ });
+ tservers.startListeningForTabletServerChanges();
+ scanning.set(true);
+ while (scanner.hasNext()) {
+ TabletLocationState locationState = scanner.next();
+ TabletState state = locationState.getState(tservers.getCurrentServers());
+ if (state != null && state != TabletState.HOSTED && TableManager.getInstance().getTableState(locationState.extent.getTableId().toString()) != TableState.OFFLINE)
+ if (!locationState.extent.equals(Constants.ROOT_TABLET_EXTENT))
+ System.out.println(locationState + " is " + state + " #walogs:" + locationState.walogs.size());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
index e16bd06,0000000..cf1a065
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
@@@ -1,62 -1,0 +1,59 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.TokenProperty;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.security.handler.Authenticator;
+import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+
+/**
+ *
+ */
+public class LoginProperties {
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ AccumuloConfiguration config = ServerConfiguration.getSystemConfiguration(HdfsZooInstance.getInstance());
+ Authenticator authenticator = AccumuloClassLoader.getClassLoader().loadClass(config.get(Property.INSTANCE_SECURITY_AUTHENTICATOR))
+ .asSubclass(Authenticator.class).newInstance();
+
+ List<Set<TokenProperty>> tokenProps = new ArrayList<Set<TokenProperty>>();
+
+ for (Class<? extends AuthenticationToken> tokenType : authenticator.getSupportedTokenTypes()) {
+ tokenProps.add(tokenType.newInstance().getProperties());
+ }
+
+ System.out.println("Supported token types for " + authenticator.getClass().getName() + " are : ");
+ for (Class<? extends AuthenticationToken> tokenType : authenticator.getSupportedTokenTypes()) {
+ System.out.println("\t" + tokenType.getName() + ", which accepts the following properties : ");
+
+ for (TokenProperty tokenProperty : tokenType.newInstance().getProperties()) {
+ System.out.println("\t\t" + tokenProperty);
+ }
+
+ System.out.println();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
index d6e0a3c,0000000..477718d
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
@@@ -1,1262 -1,0 +1,1257 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.BatchWriterImpl;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
+import org.apache.accumulo.core.client.impl.Writer;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.accumulo.core.util.FastFormat;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.StringUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.accumulo.server.trace.TraceFileSystem;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * provides a reference to the metadata table for updates by tablet servers
+ */
+public class MetadataTable extends org.apache.accumulo.core.util.MetadataTable {
-
++
+ private static final Text EMPTY_TEXT = new Text();
+ private static Map<TCredentials,Writer> metadata_tables = new HashMap<TCredentials,Writer>();
+ private static final Logger log = Logger.getLogger(MetadataTable.class);
-
++
+ private static final int SAVE_ROOT_TABLET_RETRIES = 3;
-
++
+ private MetadataTable() {
-
++
+ }
+
+ public synchronized static Writer getMetadataTable(TCredentials credentials) {
+ Writer metadataTable = metadata_tables.get(credentials);
+ if (metadataTable == null) {
+ metadataTable = new Writer(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID);
+ metadata_tables.put(credentials, metadataTable);
+ }
+ return metadataTable;
+ }
-
++
+ public static void putLockID(ZooLock zooLock, Mutation m) {
+ Constants.METADATA_LOCK_COLUMN.put(m, new Value(zooLock.getLockID().serialize(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + "/").getBytes(Constants.UTF8)));
+ }
+
+ public static void update(TCredentials credentials, Mutation m) {
+ update(credentials, null, m);
+ }
+
+ public static void update(TCredentials credentials, ZooLock zooLock, Mutation m) {
+ Writer t;
+ t = getMetadataTable(credentials);
+ if (zooLock != null)
+ putLockID(zooLock, m);
+ while (true) {
+ try {
+ t.update(m);
+ return;
+ } catch (AccumuloException e) {
+ log.error(e, e);
+ } catch (AccumuloSecurityException e) {
+ log.error(e, e);
+ } catch (ConstraintViolationException e) {
+ log.error(e, e);
+ } catch (TableNotFoundException e) {
+ log.error(e, e);
+ }
+ UtilWaitThread.sleep(1000);
+ }
-
++
+ }
-
++
+ /**
+ * new data file update function adds one data file to a tablet's list
- *
- * path should be relative to the table directory
- *
- * @param time
- * @param filesInUseByScans
- * @param zooLock
- * @param flushId
- *
++ *
++ * @param path
++ * should be relative to the table directory
+ */
+ public static void updateTabletDataFile(KeyExtent extent, String path, String mergeFile, DataFileValue dfv, String time, TCredentials credentials,
+ Set<String> filesInUseByScans, String address, ZooLock zooLock, Set<String> unusedWalLogs, TServerInstance lastLocation, long flushId) {
+ if (extent.equals(Constants.ROOT_TABLET_EXTENT)) {
+ if (unusedWalLogs != null) {
+ IZooReaderWriter zk = ZooReaderWriter.getInstance();
+ // unusedWalLogs will contain the location/name of each log in a log set
+ // the log set is stored under one of the log names, but not both
+ // find the entry under one of the names and delete it.
+ String root = getZookeeperLogLocation();
+ boolean foundEntry = false;
+ for (String entry : unusedWalLogs) {
+ String[] parts = entry.split("/");
+ String zpath = root + "/" + parts[1];
+ while (true) {
+ try {
+ if (zk.exists(zpath)) {
+ zk.recursiveDelete(zpath, NodeMissingPolicy.SKIP);
+ foundEntry = true;
+ }
+ break;
+ } catch (KeeperException e) {
+ log.error(e, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ }
+ UtilWaitThread.sleep(1000);
+ }
+ }
+ if (unusedWalLogs.size() > 0 && !foundEntry)
+ log.warn("WALog entry for root tablet did not exist " + unusedWalLogs);
+ }
+ return;
+ }
-
++
+ Mutation m = new Mutation(extent.getMetadataEntry());
-
++
+ if (dfv.getNumEntries() > 0) {
+ m.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(path), new Value(dfv.encode()));
+ Constants.METADATA_TIME_COLUMN.put(m, new Value(time.getBytes(Constants.UTF8)));
+ // stuff in this location
+ TServerInstance self = getTServerInstance(address, zooLock);
+ self.putLastLocation(m);
+ // erase the old location
+ if (lastLocation != null && !lastLocation.equals(self))
+ lastLocation.clearLastLocation(m);
+ }
+ if (unusedWalLogs != null) {
+ for (String entry : unusedWalLogs) {
+ m.putDelete(Constants.METADATA_LOG_COLUMN_FAMILY, new Text(entry));
+ }
+ }
-
++
+ for (String scanFile : filesInUseByScans)
+ m.put(Constants.METADATA_SCANFILE_COLUMN_FAMILY, new Text(scanFile), new Value(new byte[0]));
+
+ if (mergeFile != null)
+ m.putDelete(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(mergeFile));
+
+ Constants.METADATA_FLUSH_COLUMN.put(m, new Value(Long.toString(flushId).getBytes(Constants.UTF8)));
+
+ update(credentials, zooLock, m);
-
++
+ }
-
++
+ private static TServerInstance getTServerInstance(String address, ZooLock zooLock) {
+ while (true) {
+ try {
+ return new TServerInstance(address, zooLock.getSessionId());
+ } catch (KeeperException e) {
+ log.error(e, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ }
+ UtilWaitThread.sleep(1000);
+ }
+ }
+
+ public static void updateTabletFlushID(KeyExtent extent, long flushID, TCredentials credentials, ZooLock zooLock) {
+ if (!extent.isRootTablet()) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
+ Constants.METADATA_FLUSH_COLUMN.put(m, new Value(Long.toString(flushID).getBytes(Constants.UTF8)));
+ update(credentials, zooLock, m);
+ }
+ }
+
+ public static void updateTabletCompactID(KeyExtent extent, long compactID, TCredentials credentials, ZooLock zooLock) {
+ if (!extent.isRootTablet()) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
+ Constants.METADATA_COMPACT_COLUMN.put(m, new Value(Long.toString(compactID).getBytes(Constants.UTF8)));
+ update(credentials, zooLock, m);
+ }
+ }
+
+ public static void updateTabletDataFile(long tid, KeyExtent extent, Map<String,DataFileValue> estSizes, String time, TCredentials credentials, ZooLock zooLock) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
+ byte[] tidBytes = Long.toString(tid).getBytes(Constants.UTF8);
+
+ for (Entry<String,DataFileValue> entry : estSizes.entrySet()) {
+ Text file = new Text(entry.getKey());
+ m.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, file, new Value(entry.getValue().encode()));
+ m.put(Constants.METADATA_BULKFILE_COLUMN_FAMILY, file, new Value(tidBytes));
+ }
+ Constants.METADATA_TIME_COLUMN.put(m, new Value(time.getBytes(Constants.UTF8)));
+ update(credentials, zooLock, m);
+ }
+
+ public static void addTablet(KeyExtent extent, String path, TCredentials credentials, char timeType, ZooLock lock) {
+ Mutation m = extent.getPrevRowUpdateMutation();
+
+ Constants.METADATA_DIRECTORY_COLUMN.put(m, new Value(path.getBytes(Constants.UTF8)));
+ Constants.METADATA_TIME_COLUMN.put(m, new Value((timeType + "0").getBytes(Constants.UTF8)));
+
+ update(credentials, lock, m);
+ }
+
+ public static void updateTabletPrevEndRow(KeyExtent extent, TCredentials credentials) {
+ Mutation m = extent.getPrevRowUpdateMutation(); //
+ update(credentials, m);
+ }
-
++
+ /**
+ * convenience method for reading entries from the metadata table
+ */
+ public static SortedMap<KeyExtent,Text> getMetadataDirectoryEntries(SortedMap<Key,Value> entries) {
+ Key key;
+ Value val;
+ Text datafile = null;
+ Value prevRow = null;
+ KeyExtent ke;
-
++
+ SortedMap<KeyExtent,Text> results = new TreeMap<KeyExtent,Text>();
-
++
+ Text lastRowFromKey = new Text();
-
++
+ // text obj below is meant to be reused in loop for efficiency
+ Text colf = new Text();
+ Text colq = new Text();
-
++
+ for (Entry<Key,Value> entry : entries.entrySet()) {
+ key = entry.getKey();
+ val = entry.getValue();
-
++
+ if (key.compareRow(lastRowFromKey) != 0) {
+ prevRow = null;
+ datafile = null;
+ key.getRow(lastRowFromKey);
+ }
-
++
+ colf = key.getColumnFamily(colf);
+ colq = key.getColumnQualifier(colq);
-
++
+ // interpret the row id as a key extent
+ if (Constants.METADATA_DIRECTORY_COLUMN.equals(colf, colq))
+ datafile = new Text(val.toString());
-
++
+ else if (Constants.METADATA_PREV_ROW_COLUMN.equals(colf, colq))
+ prevRow = new Value(val);
-
++
+ if (datafile != null && prevRow != null) {
+ ke = new KeyExtent(key.getRow(), prevRow);
+ results.put(ke, datafile);
-
++
+ datafile = null;
+ prevRow = null;
+ }
+ }
+ return results;
+ }
+
+ public static boolean recordRootTabletLocation(String address) {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ for (int i = 0; i < SAVE_ROOT_TABLET_RETRIES; i++) {
+ try {
+ log.info("trying to write root tablet location to ZooKeeper as " + address);
+ String zRootLocPath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZROOT_TABLET_LOCATION;
+ zoo.putPersistentData(zRootLocPath, address.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
+ return true;
+ } catch (Exception e) {
+ log.error("Master: unable to save root tablet location in zookeeper. exception: " + e, e);
+ }
+ }
+ log.error("Giving up after " + SAVE_ROOT_TABLET_RETRIES + " retries");
+ return false;
+ }
+
+ public static SortedMap<String,DataFileValue> getDataFileSizes(KeyExtent extent, TCredentials credentials) {
+ TreeMap<String,DataFileValue> sizes = new TreeMap<String,DataFileValue>();
-
++
+ Scanner mdScanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ mdScanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
+ Text row = extent.getMetadataEntry();
-
++
+ Key endKey = new Key(row, Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(""));
+ endKey = endKey.followingKey(PartialKey.ROW_COLFAM);
-
++
+ mdScanner.setRange(new Range(new Key(row), endKey));
+ for (Entry<Key,Value> entry : mdScanner) {
-
++
+ if (!entry.getKey().getRow().equals(row))
+ break;
+ DataFileValue dfv = new DataFileValue(entry.getValue().get());
+ sizes.put(entry.getKey().getColumnQualifier().toString(), dfv);
+ }
-
++
+ return sizes;
+ }
-
++
+ public static void addNewTablet(KeyExtent extent, String path, TServerInstance location, Map<String,DataFileValue> datafileSizes,
+ Map<String,Long> bulkLoadedFiles, TCredentials credentials, String time, long lastFlushID, long lastCompactID, ZooLock zooLock) {
+ Mutation m = extent.getPrevRowUpdateMutation();
+
+ Constants.METADATA_DIRECTORY_COLUMN.put(m, new Value(path.getBytes(Constants.UTF8)));
+ Constants.METADATA_TIME_COLUMN.put(m, new Value(time.getBytes(Constants.UTF8)));
+ if (lastFlushID > 0)
+ Constants.METADATA_FLUSH_COLUMN.put(m, new Value(Long.toString(lastFlushID).getBytes(Constants.UTF8)));
+ if (lastCompactID > 0)
+ Constants.METADATA_COMPACT_COLUMN.put(m, new Value(Long.toString(lastCompactID).getBytes(Constants.UTF8)));
+
+ if (location != null) {
+ m.put(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY, location.asColumnQualifier(), location.asMutationValue());
+ m.putDelete(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY, location.asColumnQualifier());
+ }
-
++
+ for (Entry<String,DataFileValue> entry : datafileSizes.entrySet()) {
+ m.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(entry.getKey()), new Value(entry.getValue().encode()));
+ }
-
++
+ for (Entry<String,Long> entry : bulkLoadedFiles.entrySet()) {
+ byte[] tidBytes = Long.toString(entry.getValue()).getBytes(Constants.UTF8);
+ m.put(Constants.METADATA_BULKFILE_COLUMN_FAMILY, new Text(entry.getKey()), new Value(tidBytes));
+ }
+
+ update(credentials, zooLock, m);
+ }
+
+ public static void rollBackSplit(Text metadataEntry, Text oldPrevEndRow, TCredentials credentials, ZooLock zooLock) {
+ KeyExtent ke = new KeyExtent(metadataEntry, oldPrevEndRow);
+ Mutation m = ke.getPrevRowUpdateMutation();
+ Constants.METADATA_SPLIT_RATIO_COLUMN.putDelete(m);
+ Constants.METADATA_OLD_PREV_ROW_COLUMN.putDelete(m);
+ update(credentials, zooLock, m);
+ }
+
+ public static void splitTablet(KeyExtent extent, Text oldPrevEndRow, double splitRatio, TCredentials credentials, ZooLock zooLock) {
+ Mutation m = extent.getPrevRowUpdateMutation(); //
+
+ Constants.METADATA_SPLIT_RATIO_COLUMN.put(m, new Value(Double.toString(splitRatio).getBytes(Constants.UTF8)));
+
+ Constants.METADATA_OLD_PREV_ROW_COLUMN.put(m, KeyExtent.encodePrevEndRow(oldPrevEndRow));
+ Constants.METADATA_CHOPPED_COLUMN.putDelete(m);
+ update(credentials, zooLock, m);
+ }
+
+ public static void finishSplit(Text metadataEntry, Map<String,DataFileValue> datafileSizes, List<String> highDatafilesToRemove, TCredentials credentials,
+ ZooLock zooLock) {
+ Mutation m = new Mutation(metadataEntry);
+ Constants.METADATA_SPLIT_RATIO_COLUMN.putDelete(m);
+ Constants.METADATA_OLD_PREV_ROW_COLUMN.putDelete(m);
+ Constants.METADATA_CHOPPED_COLUMN.putDelete(m);
+
+ for (Entry<String,DataFileValue> entry : datafileSizes.entrySet()) {
+ m.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(entry.getKey()), new Value(entry.getValue().encode()));
+ }
-
++
+ for (String pathToRemove : highDatafilesToRemove) {
+ m.putDelete(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(pathToRemove));
+ }
-
++
+ update(credentials, zooLock, m);
+ }
+
+ public static void finishSplit(KeyExtent extent, Map<String,DataFileValue> datafileSizes, List<String> highDatafilesToRemove, TCredentials credentials,
+ ZooLock zooLock) {
+ finishSplit(extent.getMetadataEntry(), datafileSizes, highDatafilesToRemove, credentials, zooLock);
+ }
-
++
+ public static void replaceDatafiles(KeyExtent extent, Set<String> datafilesToDelete, Set<String> scanFiles, String path, Long compactionId,
+ DataFileValue size, TCredentials credentials, String address, TServerInstance lastLocation, ZooLock zooLock) {
+ replaceDatafiles(extent, datafilesToDelete, scanFiles, path, compactionId, size, credentials, address, lastLocation, zooLock, true);
+ }
-
++
+ public static void replaceDatafiles(KeyExtent extent, Set<String> datafilesToDelete, Set<String> scanFiles, String path, Long compactionId,
+ DataFileValue size, TCredentials credentials, String address, TServerInstance lastLocation, ZooLock zooLock, boolean insertDeleteFlags) {
+
+ if (insertDeleteFlags) {
+ // add delete flags for those paths before the data file reference is removed
+ addDeleteEntries(extent, datafilesToDelete, credentials);
+ }
-
++
+ // replace data file references to old mapfiles with the new mapfiles
+ Mutation m = new Mutation(extent.getMetadataEntry());
-
++
+ for (String pathToRemove : datafilesToDelete)
+ m.putDelete(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(pathToRemove));
-
++
+ for (String scanFile : scanFiles)
+ m.put(Constants.METADATA_SCANFILE_COLUMN_FAMILY, new Text(scanFile), new Value(new byte[0]));
+
+ if (size.getNumEntries() > 0)
+ m.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(path), new Value(size.encode()));
-
++
+ if (compactionId != null)
+ Constants.METADATA_COMPACT_COLUMN.put(m, new Value(Long.toString(compactionId).getBytes(Constants.UTF8)));
+
+ TServerInstance self = getTServerInstance(address, zooLock);
+ self.putLastLocation(m);
+
+ // remove the old location
+ if (lastLocation != null && !lastLocation.equals(self))
+ lastLocation.clearLastLocation(m);
+
+ update(credentials, zooLock, m);
+ }
+
+ public static void addDeleteEntries(KeyExtent extent, Set<String> datafilesToDelete, TCredentials credentials) {
+
+ String tableId = extent.getTableId().toString();
+
+ // TODO could use batch writer,would need to handle failure and retry like update does - ACCUMULO-1294
+ for (String pathToRemove : datafilesToDelete)
+ update(credentials, createDeleteMutation(tableId, pathToRemove));
+ }
-
++
+ public static void addDeleteEntry(String tableId, String path) {
+ update(SecurityConstants.getSystemCredentials(), createDeleteMutation(tableId, path));
+ }
-
++
+ public static Mutation createDeleteMutation(String tableId, String pathToRemove) {
+ Mutation delFlag;
+ String prefix = Constants.METADATA_DELETE_FLAG_PREFIX;
+ if (tableId.equals(Constants.METADATA_TABLE_ID))
+ prefix = Constants.METADATA_DELETE_FLAG_FOR_METADATA_PREFIX;
+
+ if (pathToRemove.startsWith("../"))
+ delFlag = new Mutation(new Text(prefix + pathToRemove.substring(2)));
+ else
+ delFlag = new Mutation(new Text(prefix + "/" + tableId + pathToRemove));
+
+ delFlag.put(EMPTY_TEXT, EMPTY_TEXT, new Value(new byte[] {}));
+ return delFlag;
+ }
+
+ public static void removeScanFiles(KeyExtent extent, Set<String> scanFiles, TCredentials credentials, ZooLock zooLock) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
-
++
+ for (String pathToRemove : scanFiles)
+ m.putDelete(Constants.METADATA_SCANFILE_COLUMN_FAMILY, new Text(pathToRemove));
-
++
+ update(credentials, zooLock, m);
+ }
+
+ private static KeyExtent fixSplit(Text table, Text metadataEntry, Text metadataPrevEndRow, Value oper, double splitRatio, TServerInstance tserver,
+ TCredentials credentials, String time, long initFlushID, long initCompactID, ZooLock lock) throws AccumuloException {
+ if (metadataPrevEndRow == null)
+ // something is wrong, this should not happen... if a tablet is split, it will always have a
+ // prev end row....
+ throw new AccumuloException("Split tablet does not have prev end row, something is amiss, extent = " + metadataEntry);
+
+ // check to see if prev tablet exist in metadata tablet
+ Key prevRowKey = new Key(new Text(KeyExtent.getMetadataEntry(table, metadataPrevEndRow)));
+
+ ScannerImpl scanner2 = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ scanner2.setRange(new Range(prevRowKey, prevRowKey.followingKey(PartialKey.ROW)));
-
++
+ if (!scanner2.iterator().hasNext()) {
+ log.info("Rolling back incomplete split " + metadataEntry + " " + metadataPrevEndRow);
+ rollBackSplit(metadataEntry, KeyExtent.decodePrevEndRow(oper), credentials, lock);
+ return new KeyExtent(metadataEntry, KeyExtent.decodePrevEndRow(oper));
+ } else {
+ log.info("Finishing incomplete split " + metadataEntry + " " + metadataPrevEndRow);
+
+ List<String> highDatafilesToRemove = new ArrayList<String>();
+
+ Scanner scanner3 = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ Key rowKey = new Key(metadataEntry);
+
+ SortedMap<String,DataFileValue> origDatafileSizes = new TreeMap<String,DataFileValue>();
+ SortedMap<String,DataFileValue> highDatafileSizes = new TreeMap<String,DataFileValue>();
+ SortedMap<String,DataFileValue> lowDatafileSizes = new TreeMap<String,DataFileValue>();
+ scanner3.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
+ scanner3.setRange(new Range(rowKey, rowKey.followingKey(PartialKey.ROW)));
+
+ for (Entry<Key,Value> entry : scanner3) {
+ if (entry.getKey().compareColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY) == 0) {
+ origDatafileSizes.put(entry.getKey().getColumnQualifier().toString(), new DataFileValue(entry.getValue().get()));
+ }
+ }
+
+ splitDatafiles(table, metadataPrevEndRow, splitRatio, new HashMap<String,FileUtil.FileInfo>(), origDatafileSizes, lowDatafileSizes, highDatafileSizes,
+ highDatafilesToRemove);
+
+ MetadataTable.finishSplit(metadataEntry, highDatafileSizes, highDatafilesToRemove, credentials, lock);
+
+ return new KeyExtent(metadataEntry, KeyExtent.encodePrevEndRow(metadataPrevEndRow));
+ }
+
+
+ }
-
++
+ public static void splitDatafiles(Text table, Text midRow, double splitRatio, Map<String,FileUtil.FileInfo> firstAndLastRows,
+ SortedMap<String,DataFileValue> datafiles, SortedMap<String,DataFileValue> lowDatafileSizes, SortedMap<String,DataFileValue> highDatafileSizes,
+ List<String> highDatafilesToRemove) {
-
++
+ for (Entry<String,DataFileValue> entry : datafiles.entrySet()) {
-
++
+ Text firstRow = null;
+ Text lastRow = null;
-
++
+ boolean rowsKnown = false;
-
++
+ FileUtil.FileInfo mfi = firstAndLastRows.get(entry.getKey());
-
++
+ if (mfi != null) {
+ firstRow = mfi.getFirstRow();
+ lastRow = mfi.getLastRow();
+ rowsKnown = true;
+ }
-
++
+ if (rowsKnown && firstRow.compareTo(midRow) > 0) {
+ // only in high
+ long highSize = entry.getValue().getSize();
+ long highEntries = entry.getValue().getNumEntries();
+ highDatafileSizes.put(entry.getKey(), new DataFileValue(highSize, highEntries, entry.getValue().getTime()));
+ } else if (rowsKnown && lastRow.compareTo(midRow) <= 0) {
+ // only in low
+ long lowSize = entry.getValue().getSize();
+ long lowEntries = entry.getValue().getNumEntries();
+ lowDatafileSizes.put(entry.getKey(), new DataFileValue(lowSize, lowEntries, entry.getValue().getTime()));
-
++
+ highDatafilesToRemove.add(entry.getKey());
+ } else {
+ long lowSize = (long) Math.floor((entry.getValue().getSize() * splitRatio));
+ long lowEntries = (long) Math.floor((entry.getValue().getNumEntries() * splitRatio));
+ lowDatafileSizes.put(entry.getKey(), new DataFileValue(lowSize, lowEntries, entry.getValue().getTime()));
-
++
+ long highSize = (long) Math.ceil((entry.getValue().getSize() * (1.0 - splitRatio)));
+ long highEntries = (long) Math.ceil((entry.getValue().getNumEntries() * (1.0 - splitRatio)));
+ highDatafileSizes.put(entry.getKey(), new DataFileValue(highSize, highEntries, entry.getValue().getTime()));
+ }
+ }
+ }
+
+ public static KeyExtent fixSplit(Text metadataEntry, SortedMap<ColumnFQ,Value> columns, TServerInstance tserver, TCredentials credentials, ZooLock lock)
+ throws AccumuloException {
+ log.info("Incomplete split " + metadataEntry + " attempting to fix");
+
+ Value oper = columns.get(Constants.METADATA_OLD_PREV_ROW_COLUMN);
-
++
+ if (columns.get(Constants.METADATA_SPLIT_RATIO_COLUMN) == null) {
+ throw new IllegalArgumentException("Metadata entry does not have split ratio (" + metadataEntry + ")");
+ }
+
+ double splitRatio = Double.parseDouble(new String(columns.get(Constants.METADATA_SPLIT_RATIO_COLUMN).get(), Constants.UTF8));
+
+ Value prevEndRowIBW = columns.get(Constants.METADATA_PREV_ROW_COLUMN);
-
++
+ if (prevEndRowIBW == null) {
+ throw new IllegalArgumentException("Metadata entry does not have prev row (" + metadataEntry + ")");
+ }
-
++
+ Value time = columns.get(Constants.METADATA_TIME_COLUMN);
-
++
+ if (time == null) {
+ throw new IllegalArgumentException("Metadata entry does not have time (" + metadataEntry + ")");
+ }
-
++
+ Value flushID = columns.get(Constants.METADATA_FLUSH_COLUMN);
+ long initFlushID = -1;
+ if (flushID != null)
+ initFlushID = Long.parseLong(flushID.toString());
-
++
+ Value compactID = columns.get(Constants.METADATA_COMPACT_COLUMN);
+ long initCompactID = -1;
+ if (compactID != null)
+ initCompactID = Long.parseLong(compactID.toString());
-
++
+ Text metadataPrevEndRow = KeyExtent.decodePrevEndRow(prevEndRowIBW);
-
++
+ Text table = (new KeyExtent(metadataEntry, (Text) null)).getTableId();
-
++
+ return fixSplit(table, metadataEntry, metadataPrevEndRow, oper, splitRatio, tserver, credentials, time.toString(), initFlushID, initCompactID, lock);
+ }
+
+ public static void deleteTable(String tableId, boolean insertDeletes, TCredentials credentials, ZooLock lock) throws AccumuloException {
+ Scanner ms = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ Text tableIdText = new Text(tableId);
+ BatchWriter bw = new BatchWriterImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, new BatchWriterConfig().setMaxMemory(1000000)
+ .setMaxLatency(120000l, TimeUnit.MILLISECONDS).setMaxWriteThreads(2));
+
+ // scan metadata for our table and delete everything we find
+ Mutation m = null;
+ ms.setRange(new KeyExtent(tableIdText, null, null).toMetadataRange());
-
++
+ // insert deletes before deleting data from !METADATA... this makes the code fault tolerant
+ if (insertDeletes) {
-
++
+ ms.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
+ Constants.METADATA_DIRECTORY_COLUMN.fetch(ms);
+
+ for (Entry<Key,Value> cell : ms) {
+ Key key = cell.getKey();
-
++
+ if (key.getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
+ String relPath = key.getColumnQualifier().toString();
+ // only insert deletes for files owned by this table
+ if (!relPath.startsWith("../"))
+ bw.addMutation(createDeleteMutation(tableId, relPath));
+ }
-
++
+ if (Constants.METADATA_DIRECTORY_COLUMN.hasColumns(key)) {
+ bw.addMutation(createDeleteMutation(tableId, cell.getValue().toString()));
+ }
+ }
-
++
+ bw.flush();
-
++
+ ms.clearColumns();
+ }
-
++
+ for (Entry<Key,Value> cell : ms) {
+ Key key = cell.getKey();
-
++
+ if (m == null) {
+ m = new Mutation(key.getRow());
+ if (lock != null)
+ putLockID(lock, m);
+ }
-
++
+ if (key.getRow().compareTo(m.getRow(), 0, m.getRow().length) != 0) {
+ bw.addMutation(m);
+ m = new Mutation(key.getRow());
+ if (lock != null)
+ putLockID(lock, m);
+ }
+ m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
+ }
-
++
+ if (m != null)
+ bw.addMutation(m);
-
++
+ bw.close();
+ }
-
++
+ public static class LogEntry {
+ public KeyExtent extent;
+ public long timestamp;
+ public String server;
+ public String filename;
+ public int tabletId;
+ public Collection<String> logSet;
+
+ @Override
+ public String toString() {
+ return extent.toString() + " " + filename + " (" + tabletId + ")";
+ }
-
++
+ public String getName() {
+ return server + "/" + filename;
+ }
-
++
+ public byte[] toBytes() throws IOException {
+ DataOutputBuffer out = new DataOutputBuffer();
+ extent.write(out);
+ out.writeLong(timestamp);
+ out.writeUTF(server);
+ out.writeUTF(filename);
+ out.write(tabletId);
+ out.write(logSet.size());
+ for (String s : logSet) {
+ out.writeUTF(s);
+ }
+ return Arrays.copyOf(out.getData(), out.getLength());
+ }
-
++
+ public void fromBytes(byte bytes[]) throws IOException {
+ DataInputBuffer inp = new DataInputBuffer();
+ inp.reset(bytes, bytes.length);
+ extent = new KeyExtent();
+ extent.readFields(inp);
+ timestamp = inp.readLong();
+ server = inp.readUTF();
+ filename = inp.readUTF();
+ tabletId = inp.read();
+ int count = inp.read();
+ ArrayList<String> logSet = new ArrayList<String>(count);
+ for (int i = 0; i < count; i++)
+ logSet.add(inp.readUTF());
+ this.logSet = logSet;
+ }
-
++
+ }
-
++
+ private static String getZookeeperLogLocation() {
+ return ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZROOT_TABLET_WALOGS;
+ }
+
+ public static void addLogEntry(TCredentials credentials, LogEntry entry, ZooLock zooLock) {
+ if (entry.extent.isRootTablet()) {
+ String root = getZookeeperLogLocation();
+ while (true) {
+ try {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ if (zoo.isLockHeld(zooLock.getLockID()))
+ zoo.putPersistentData(root + "/" + entry.filename, entry.toBytes(), NodeExistsPolicy.OVERWRITE);
+ break;
+ } catch (KeeperException e) {
+ log.error(e, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ } catch (IOException e) {
+ log.error(e, e);
+ }
+ UtilWaitThread.sleep(1000);
+ }
+ } else {
+ String value = StringUtil.join(entry.logSet, ";") + "|" + entry.tabletId;
+ Mutation m = new Mutation(entry.extent.getMetadataEntry());
+ m.put(Constants.METADATA_LOG_COLUMN_FAMILY, new Text(entry.server + "/" + entry.filename), new Value(value.getBytes(Constants.UTF8)));
+ update(credentials, zooLock, m);
+ }
+ }
-
++
+ public static LogEntry entryFromKeyValue(Key key, Value value) {
+ MetadataTable.LogEntry e = new MetadataTable.LogEntry();
+ e.extent = new KeyExtent(key.getRow(), EMPTY_TEXT);
+ String[] parts = key.getColumnQualifier().toString().split("/");
+ e.server = parts[0];
+ e.filename = parts[1];
+ parts = value.toString().split("\\|");
+ e.tabletId = Integer.parseInt(parts[1]);
+ e.logSet = Arrays.asList(parts[0].split(";"));
+ e.timestamp = key.getTimestamp();
+ return e;
+ }
+
+ public static Pair<List<LogEntry>,SortedMap<String,DataFileValue>> getFileAndLogEntries(TCredentials credentials, KeyExtent extent) throws KeeperException,
+ InterruptedException, IOException {
+ ArrayList<LogEntry> result = new ArrayList<LogEntry>();
+ TreeMap<String,DataFileValue> sizes = new TreeMap<String,DataFileValue>();
+
+ if (extent.isRootTablet()) {
+ getRootLogEntries(result);
+ FileSystem fs = TraceFileSystem.wrap(FileUtil.getFileSystem(CachedConfiguration.getInstance(), ServerConfiguration.getSiteConfiguration()));
+ FileStatus[] files = fs.listStatus(new Path(ServerConstants.getRootTabletDir()));
-
++
+ for (FileStatus fileStatus : files) {
+ if (fileStatus.getPath().toString().endsWith("_tmp")) {
+ continue;
+ }
+ DataFileValue dfv = new DataFileValue(0, 0);
+ sizes.put(Constants.ZROOT_TABLET + "/" + fileStatus.getPath().getName(), dfv);
+ }
-
++
+ } else {
+ Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ scanner.fetchColumnFamily(Constants.METADATA_LOG_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
+ scanner.setRange(extent.toMetadataRange());
-
++
+ for (Entry<Key,Value> entry : scanner) {
+ if (!entry.getKey().getRow().equals(extent.getMetadataEntry())) {
+ throw new RuntimeException("Unexpected row " + entry.getKey().getRow() + " expected " + extent.getMetadataEntry());
+ }
-
++
+ if (entry.getKey().getColumnFamily().equals(Constants.METADATA_LOG_COLUMN_FAMILY)) {
+ result.add(entryFromKeyValue(entry.getKey(), entry.getValue()));
+ } else if (entry.getKey().getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
+ DataFileValue dfv = new DataFileValue(entry.getValue().get());
+ sizes.put(entry.getKey().getColumnQualifier().toString(), dfv);
+ } else {
+ throw new RuntimeException("Unexpected col fam " + entry.getKey().getColumnFamily());
+ }
+ }
+ }
-
++
+ return new Pair<List<LogEntry>,SortedMap<String,DataFileValue>>(result, sizes);
+ }
+
+ public static List<LogEntry> getLogEntries(TCredentials credentials, KeyExtent extent) throws IOException, KeeperException, InterruptedException {
+ log.info("Scanning logging entries for " + extent);
+ ArrayList<LogEntry> result = new ArrayList<LogEntry>();
+ if (extent.equals(Constants.ROOT_TABLET_EXTENT)) {
+ log.info("Getting logs for root tablet from zookeeper");
+ getRootLogEntries(result);
+ } else {
+ log.info("Scanning metadata for logs used for tablet " + extent);
+ Scanner scanner = getTabletLogScanner(credentials, extent);
+ Text pattern = extent.getMetadataEntry();
+ for (Entry<Key,Value> entry : scanner) {
+ Text row = entry.getKey().getRow();
+ if (entry.getKey().getColumnFamily().equals(Constants.METADATA_LOG_COLUMN_FAMILY)) {
+ if (row.equals(pattern)) {
+ result.add(entryFromKeyValue(entry.getKey(), entry.getValue()));
+ }
+ }
+ }
+ }
-
++
+ Collections.sort(result, new Comparator<LogEntry>() {
+ @Override
+ public int compare(LogEntry o1, LogEntry o2) {
+ long diff = o1.timestamp - o2.timestamp;
+ if (diff < 0)
+ return -1;
+ if (diff > 0)
+ return 1;
+ return 0;
+ }
+ });
+ log.info("Returning logs " + result + " for extent " + extent);
+ return result;
+ }
-
++
+ private static void getRootLogEntries(ArrayList<LogEntry> result) throws KeeperException, InterruptedException, IOException {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ String root = getZookeeperLogLocation();
+ // there's a little race between getting the children and fetching
+ // the data. The log can be removed in between.
+ while (true) {
+ result.clear();
+ for (String child : zoo.getChildren(root)) {
+ LogEntry e = new LogEntry();
+ try {
+ e.fromBytes(zoo.getData(root + "/" + child, null));
+ result.add(e);
+ } catch (KeeperException.NoNodeException ex) {
+ continue;
+ }
+ }
+ break;
+ }
+ }
+
+ private static Scanner getTabletLogScanner(TCredentials credentials, KeyExtent extent) {
+ Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ scanner.fetchColumnFamily(Constants.METADATA_LOG_COLUMN_FAMILY);
+ Text start = extent.getMetadataEntry();
+ Key endKey = new Key(start, Constants.METADATA_LOG_COLUMN_FAMILY);
+ endKey = endKey.followingKey(PartialKey.ROW_COLFAM);
+ scanner.setRange(new Range(new Key(start), endKey));
+ return scanner;
+ }
-
++
+ static class LogEntryIterator implements Iterator<LogEntry> {
-
++
+ Iterator<LogEntry> rootTabletEntries = null;
+ Iterator<Entry<Key,Value>> metadataEntries = null;
+
+ LogEntryIterator(TCredentials creds) throws IOException, KeeperException, InterruptedException {
+ rootTabletEntries = getLogEntries(creds, Constants.ROOT_TABLET_EXTENT).iterator();
+ try {
+ Scanner scanner = HdfsZooInstance.getInstance().getConnector(creds.getPrincipal(), CredentialHelper.extractToken(creds))
+ .createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ scanner.fetchColumnFamily(Constants.METADATA_LOG_COLUMN_FAMILY);
+ metadataEntries = scanner.iterator();
+ } catch (Exception ex) {
+ throw new IOException(ex);
+ }
+ }
-
++
+ @Override
+ public boolean hasNext() {
+ return rootTabletEntries.hasNext() || metadataEntries.hasNext();
+ }
-
++
+ @Override
+ public LogEntry next() {
+ if (rootTabletEntries.hasNext()) {
+ return rootTabletEntries.next();
+ }
+ Entry<Key,Value> entry = metadataEntries.next();
+ return entryFromKeyValue(entry.getKey(), entry.getValue());
+ }
-
++
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ public static Iterator<LogEntry> getLogEntries(TCredentials creds) throws IOException, KeeperException, InterruptedException {
+ return new LogEntryIterator(creds);
+ }
-
++
+ public static void removeUnusedWALEntries(KeyExtent extent, List<LogEntry> logEntries, ZooLock zooLock) {
+ if (extent.isRootTablet()) {
+ for (LogEntry entry : logEntries) {
+ String root = getZookeeperLogLocation();
+ while (true) {
+ try {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ if (zoo.isLockHeld(zooLock.getLockID()))
+ zoo.recursiveDelete(root + "/" + entry.filename, NodeMissingPolicy.SKIP);
+ break;
+ } catch (Exception e) {
+ log.error(e, e);
+ }
+ UtilWaitThread.sleep(1000);
+ }
+ }
+ } else {
+ Mutation m = new Mutation(extent.getMetadataEntry());
+ for (LogEntry entry : logEntries) {
+ m.putDelete(Constants.METADATA_LOG_COLUMN_FAMILY, new Text(entry.server + "/" + entry.filename));
+ }
+ update(SecurityConstants.getSystemCredentials(), zooLock, m);
+ }
+ }
-
++
+ private static void getFiles(Set<String> files, Map<Key,Value> tablet, String srcTableId) {
+ for (Entry<Key,Value> entry : tablet.entrySet()) {
+ if (entry.getKey().getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
+ String cf = entry.getKey().getColumnQualifier().toString();
+ if (srcTableId != null && !cf.startsWith("../"))
+ cf = "../" + srcTableId + entry.getKey().getColumnQualifier();
+ files.add(cf);
+ }
+ }
+ }
-
++
+ private static Mutation createCloneMutation(String srcTableId, String tableId, Map<Key,Value> tablet) {
-
++
+ KeyExtent ke = new KeyExtent(tablet.keySet().iterator().next().getRow(), (Text) null);
+ Mutation m = new Mutation(KeyExtent.getMetadataEntry(new Text(tableId), ke.getEndRow()));
-
++
+ for (Entry<Key,Value> entry : tablet.entrySet()) {
+ if (entry.getKey().getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
+ String cf = entry.getKey().getColumnQualifier().toString();
+ if (!cf.startsWith("../"))
+ cf = "../" + srcTableId + entry.getKey().getColumnQualifier();
+ m.put(entry.getKey().getColumnFamily(), new Text(cf), entry.getValue());
+ } else if (entry.getKey().getColumnFamily().equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY)) {
+ m.put(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY, entry.getKey().getColumnQualifier(), entry.getValue());
+ } else if (entry.getKey().getColumnFamily().equals(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY)) {
+ // skip
+ } else {
+ m.put(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier(), entry.getValue());
+ }
+ }
+ return m;
+ }
-
++
+ private static Scanner createCloneScanner(String tableId, Connector conn) throws TableNotFoundException {
+ Scanner mscanner = new IsolatedScanner(conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS));
+ mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+ mscanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
+ mscanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
+ mscanner.fetchColumnFamily(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY);
+ mscanner.fetchColumnFamily(Constants.METADATA_CLONED_COLUMN_FAMILY);
+ Constants.METADATA_PREV_ROW_COLUMN.fetch(mscanner);
+ Constants.METADATA_TIME_COLUMN.fetch(mscanner);
+ return mscanner;
+ }
-
++
+ static void initializeClone(String srcTableId, String tableId, Connector conn, BatchWriter bw) throws TableNotFoundException, MutationsRejectedException {
+ TabletIterator ti = new TabletIterator(createCloneScanner(srcTableId, conn), new KeyExtent(new Text(srcTableId), null, null).toMetadataRange(), true, true);
-
++
+ if (!ti.hasNext())
+ throw new RuntimeException(" table deleted during clone? srcTableId = " + srcTableId);
-
++
+ while (ti.hasNext())
+ bw.addMutation(createCloneMutation(srcTableId, tableId, ti.next()));
-
++
+ bw.flush();
+ }
-
++
+ static int compareEndRows(Text endRow1, Text endRow2) {
+ return new KeyExtent(new Text("0"), endRow1, null).compareTo(new KeyExtent(new Text("0"), endRow2, null));
+ }
-
++
+ static int checkClone(String srcTableId, String tableId, Connector conn, BatchWriter bw) throws TableNotFoundException, MutationsRejectedException {
+ TabletIterator srcIter = new TabletIterator(createCloneScanner(srcTableId, conn), new KeyExtent(new Text(srcTableId), null, null).toMetadataRange(), true,
+ true);
+ TabletIterator cloneIter = new TabletIterator(createCloneScanner(tableId, conn), new KeyExtent(new Text(tableId), null, null).toMetadataRange(), true, true);
-
++
+ if (!cloneIter.hasNext() || !srcIter.hasNext())
+ throw new RuntimeException(" table deleted during clone? srcTableId = " + srcTableId + " tableId=" + tableId);
-
++
+ int rewrites = 0;
-
++
+ while (cloneIter.hasNext()) {
+ Map<Key,Value> cloneTablet = cloneIter.next();
+ Text cloneEndRow = new KeyExtent(cloneTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
+ HashSet<String> cloneFiles = new HashSet<String>();
-
++
+ boolean cloneSuccessful = false;
+ for (Entry<Key,Value> entry : cloneTablet.entrySet()) {
+ if (entry.getKey().getColumnFamily().equals(Constants.METADATA_CLONED_COLUMN_FAMILY)) {
+ cloneSuccessful = true;
+ break;
+ }
+ }
-
++
+ if (!cloneSuccessful)
+ getFiles(cloneFiles, cloneTablet, null);
-
++
+ List<Map<Key,Value>> srcTablets = new ArrayList<Map<Key,Value>>();
+ Map<Key,Value> srcTablet = srcIter.next();
+ srcTablets.add(srcTablet);
-
++
+ Text srcEndRow = new KeyExtent(srcTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
-
++
+ int cmp = compareEndRows(cloneEndRow, srcEndRow);
+ if (cmp < 0)
+ throw new TabletIterator.TabletDeletedException("Tablets deleted from src during clone : " + cloneEndRow + " " + srcEndRow);
-
++
+ HashSet<String> srcFiles = new HashSet<String>();
+ if (!cloneSuccessful)
+ getFiles(srcFiles, srcTablet, srcTableId);
-
++
+ while (cmp > 0) {
+ srcTablet = srcIter.next();
+ srcTablets.add(srcTablet);
+ srcEndRow = new KeyExtent(srcTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
+ cmp = compareEndRows(cloneEndRow, srcEndRow);
+ if (cmp < 0)
+ throw new TabletIterator.TabletDeletedException("Tablets deleted from src during clone : " + cloneEndRow + " " + srcEndRow);
-
++
+ if (!cloneSuccessful)
+ getFiles(srcFiles, srcTablet, srcTableId);
+ }
-
++
+ if (cloneSuccessful)
+ continue;
-
++
+ if (!srcFiles.containsAll(cloneFiles)) {
+ // delete existing cloned tablet entry
+ Mutation m = new Mutation(cloneTablet.keySet().iterator().next().getRow());
-
++
+ for (Entry<Key,Value> entry : cloneTablet.entrySet()) {
+ Key k = entry.getKey();
+ m.putDelete(k.getColumnFamily(), k.getColumnQualifier(), k.getTimestamp());
+ }
-
++
+ bw.addMutation(m);
-
++
+ for (Map<Key,Value> st : srcTablets)
+ bw.addMutation(createCloneMutation(srcTableId, tableId, st));
-
++
+ rewrites++;
+ } else {
+ // write out marker that this tablet was successfully cloned
+ Mutation m = new Mutation(cloneTablet.keySet().iterator().next().getRow());
+ m.put(Constants.METADATA_CLONED_COLUMN_FAMILY, new Text(""), new Value("OK".getBytes(Constants.UTF8)));
+ bw.addMutation(m);
+ }
+ }
-
++
+ bw.flush();
+ return rewrites;
+ }
-
++
+ public static void cloneTable(Instance instance, String srcTableId, String tableId) throws Exception {
+
+ Connector conn = instance.getConnector(SecurityConstants.SYSTEM_PRINCIPAL, SecurityConstants.getSystemToken());
+ BatchWriter bw = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig());
+
+ while (true) {
-
++
+ try {
+ initializeClone(srcTableId, tableId, conn, bw);
-
++
+ // the following loop looks changes in the file that occurred during the copy.. if files were dereferenced then they could have been GCed
-
++
+ while (true) {
+ int rewrites = checkClone(srcTableId, tableId, conn, bw);
-
++
+ if (rewrites == 0)
+ break;
+ }
-
++
+ bw.flush();
+ break;
-
++
+ } catch (TabletIterator.TabletDeletedException tde) {
+ // tablets were merged in the src table
+ bw.flush();
-
++
+ // delete what we have cloned and try again
+ deleteTable(tableId, false, SecurityConstants.getSystemCredentials(), null);
-
++
+ log.debug("Tablets merged in table " + srcTableId + " while attempting to clone, trying again");
-
++
+ UtilWaitThread.sleep(100);
+ }
+ }
-
++
+ // delete the clone markers and create directory entries
+ Scanner mscanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+ mscanner.fetchColumnFamily(Constants.METADATA_CLONED_COLUMN_FAMILY);
-
++
+ int dirCount = 0;
-
++
+ for (Entry<Key,Value> entry : mscanner) {
+ Key k = entry.getKey();
+ Mutation m = new Mutation(k.getRow());
+ m.putDelete(k.getColumnFamily(), k.getColumnQualifier());
+ Constants.METADATA_DIRECTORY_COLUMN.put(m, new Value(FastFormat.toZeroPaddedString(dirCount++, 8, 16, "/c-".getBytes(Constants.UTF8))));
+ bw.addMutation(m);
+ }
-
++
+ bw.close();
-
++
+ }
-
++
+ public static void chopped(KeyExtent extent, ZooLock zooLock) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
+ Constants.METADATA_CHOPPED_COLUMN.put(m, new Value("chopped".getBytes(Constants.UTF8)));
+ update(SecurityConstants.getSystemCredentials(), zooLock, m);
+ }
-
++
+ public static void removeBulkLoadEntries(Connector conn, String tableId, long tid) throws Exception {
+ Scanner mscanner = new IsolatedScanner(conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS));
+ mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+ mscanner.fetchColumnFamily(Constants.METADATA_BULKFILE_COLUMN_FAMILY);
+ BatchWriter bw = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig());
+ for (Entry<Key,Value> entry : mscanner) {
+ log.debug("Looking at entry " + entry + " with tid " + tid);
+ if (Long.parseLong(entry.getValue().toString()) == tid) {
+ log.debug("deleting entry " + entry);
+ Mutation m = new Mutation(entry.getKey().getRow());
+ m.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier());
+ bw.addMutation(m);
+ }
+ }
+ bw.close();
+ }
-
++
+ public static List<String> getBulkFilesLoaded(Connector conn, KeyExtent extent, long tid) {
+ List<String> result = new ArrayList<String>();
+ try {
+ Scanner mscanner = new IsolatedScanner(conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS));
+ mscanner.setRange(extent.toMetadataRange());
+ mscanner.fetchColumnFamily(Constants.METADATA_BULKFILE_COLUMN_FAMILY);
+ for (Entry<Key,Value> entry : mscanner) {
+ if (Long.parseLong(entry.getValue().toString()) == tid) {
+ result.add(entry.getKey().getColumnQualifier().toString());
+ }
+ }
+ return result;
+ } catch (TableNotFoundException ex) {
+ // unlikely
+ throw new RuntimeException("Onos! teh metadata table has vanished!!");
+ }
+ }
+
+ public static Map<String,Long> getBulkFilesLoaded(TCredentials credentials, KeyExtent extent) {
+ return getBulkFilesLoaded(credentials, extent.getMetadataEntry());
+ }
+
+ public static Map<String,Long> getBulkFilesLoaded(TCredentials credentials, Text metadataRow) {
+
+ Map<String,Long> ret = new HashMap<String,Long>();
-
++
+ Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ scanner.setRange(new Range(metadataRow));
+ scanner.fetchColumnFamily(Constants.METADATA_BULKFILE_COLUMN_FAMILY);
+ for (Entry<Key,Value> entry : scanner) {
+ String file = entry.getKey().getColumnQualifier().toString();
+ Long tid = Long.parseLong(entry.getValue().toString());
-
++
+ ret.put(file, tid);
+ }
+ return ret;
+ }
+
+ public static void addBulkLoadInProgressFlag(String path) {
-
++
+ Mutation m = new Mutation(Constants.METADATA_BLIP_FLAG_PREFIX + path);
+ m.put(EMPTY_TEXT, EMPTY_TEXT, new Value(new byte[] {}));
-
++
+ update(SecurityConstants.getSystemCredentials(), m);
+ }
-
++
+ public static void removeBulkLoadInProgressFlag(String path) {
-
++
+ Mutation m = new Mutation(Constants.METADATA_BLIP_FLAG_PREFIX + path);
+ m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
-
++
+ update(SecurityConstants.getSystemCredentials(), m);
+ }
+
+ /**
+ * During an upgrade from Accumulo 1.4 -> 1.5, we need to move deletion requests for files under the !METADATA table to the root tablet.
+ */
+ public static void moveMetaDeleteMarkers(Instance instance, TCredentials creds) {
+ // move delete markers from the normal delete keyspace to the root tablet delete keyspace if the files are for the !METADATA table
+ Scanner scanner = new ScannerImpl(instance, creds, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ scanner.setRange(new Range(Constants.METADATA_DELETES_KEYSPACE));
+ for (Entry<Key,Value> entry : scanner) {
+ String row = entry.getKey().getRow().toString();
+ if (row.startsWith(Constants.METADATA_DELETE_FLAG_PREFIX + "/" + Constants.METADATA_TABLE_ID)) {
+ String filename = row.substring(Constants.METADATA_DELETE_FLAG_PREFIX.length());
+ // add the new entry first
+ log.info("Moving " + filename + " marker to the root tablet");
+ Mutation m = new Mutation(Constants.METADATA_DELETE_FLAG_FOR_METADATA_PREFIX + filename);
+ m.put(new byte[]{}, new byte[]{}, new byte[]{});
+ update(creds, m);
+ // remove the old entry
+ m = new Mutation(entry.getKey().getRow());
+ m.putDelete(new byte[]{}, new byte[]{});
+ update(creds, m);
+ } else {
+ break;
+ }
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
index 8985b27,0000000..8e41db7
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
@@@ -1,127 -1,0 +1,123 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.Stack;
+
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.DefaultHandler;
+
+import com.beust.jcommander.Parameter;
+
+public class RestoreZookeeper {
+
+ private static class Restore extends DefaultHandler {
+ IZooReaderWriter zk = null;
+ Stack<String> cwd = new Stack<String>();
+ boolean overwrite = false;
+
+ Restore(IZooReaderWriter zk, boolean overwrite) {
+ this.zk = zk;
+ this.overwrite = overwrite;
+ }
+
+ @Override
+ public void startElement(String uri, String localName, String name, Attributes attributes) throws SAXException {
+ if ("node".equals(name)) {
+ String child = attributes.getValue("name");
+ if (child == null)
+ throw new RuntimeException("name attribute not set");
+ String encoding = attributes.getValue("encoding");
+ String value = attributes.getValue("value");
+ if (value == null)
+ value = "";
+ String path = cwd.lastElement() + "/" + child;
+ create(path, value, encoding);
+ cwd.push(path);
+ } else if ("dump".equals(name)) {
+ String root = attributes.getValue("root");
+ if (root.equals("/"))
+ cwd.push("");
+ else
+ cwd.push(root);
+ create(root, "", "utf-8");
+ }
+ }
+
+ @Override
+ public void endElement(String uri, String localName, String name) throws SAXException {
+ cwd.pop();
+ }
+
+ private void create(String path, String value, String encoding) {
+ byte[] data = value.getBytes(Constants.UTF8);
+ if ("base64".equals(encoding))
+ data = Base64.decodeBase64(data);
+ try {
+ try {
+ zk.putPersistentData(path, data, overwrite ? NodeExistsPolicy.OVERWRITE : NodeExistsPolicy.FAIL);
+ } catch (KeeperException e) {
+ if (e.code().equals(KeeperException.Code.NODEEXISTS))
+ throw new RuntimeException(path + " exists. Remove it first.");
+ throw e;
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static class Opts extends Help {
+ @Parameter(names={"-z", "--keepers"})
+ String keepers = "localhost:2181";
+ @Parameter(names="--overwrite")
+ boolean overwrite = false;
+ @Parameter(names="--file")
+ String file;
+ }
+
- /**
- * @param args
- * @throws Exception
- */
+ public static void main(String[] args) throws Exception {
+ Logger.getRootLogger().setLevel(Level.WARN);
+ Opts opts = new Opts();
+ opts.parseArgs(RestoreZookeeper.class.getName(), args);
+
+ InputStream in = System.in;
+ if (opts.file != null) {
+ in = new FileInputStream(opts.file);
+ }
+
+ SAXParserFactory factory = SAXParserFactory.newInstance();
+ SAXParser parser = factory.newSAXParser();
+ parser.parse(in, new Restore(ZooReaderWriter.getInstance(), opts.overwrite));
+ in.close();
+ }
+}
[21/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
index ad3d615,0000000..b9b68bb
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
@@@ -1,3620 -1,0 +1,3614 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.tabletserver;
+
+import static org.apache.accumulo.server.problems.ProblemType.TABLET_LOAD;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.lang.reflect.Field;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TimerTask;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.UUID;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.RunnableFuture;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.management.ObjectName;
+import javax.management.StandardMBean;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
+import org.apache.accumulo.core.client.impl.TabletType;
+import org.apache.accumulo.core.client.impl.Translator;
++import org.apache.accumulo.core.client.impl.Translators;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
- import org.apache.accumulo.core.client.impl.Translators;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.constraints.Constraint.Environment;
+import org.apache.accumulo.core.constraints.Violations;
+import org.apache.accumulo.core.data.Column;
+import org.apache.accumulo.core.data.ConstraintViolationSummary;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.thrift.InitialMultiScan;
+import org.apache.accumulo.core.data.thrift.InitialScan;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.data.thrift.MapFileInfo;
+import org.apache.accumulo.core.data.thrift.MultiScanResult;
+import org.apache.accumulo.core.data.thrift.ScanResult;
+import org.apache.accumulo.core.data.thrift.TColumn;
+import org.apache.accumulo.core.data.thrift.TKey;
+import org.apache.accumulo.core.data.thrift.TKeyExtent;
+import org.apache.accumulo.core.data.thrift.TKeyValue;
+import org.apache.accumulo.core.data.thrift.TMutation;
+import org.apache.accumulo.core.data.thrift.TRange;
+import org.apache.accumulo.core.data.thrift.UpdateErrors;
+import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.iterators.IterationInterruptedException;
+import org.apache.accumulo.core.master.thrift.Compacting;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.master.thrift.TableInfo;
+import org.apache.accumulo.core.master.thrift.TabletLoadState;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
+import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
+import org.apache.accumulo.core.tabletserver.thrift.NoSuchScanIDException;
+import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
+import org.apache.accumulo.core.tabletserver.thrift.ScanState;
+import org.apache.accumulo.core.tabletserver.thrift.ScanType;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Processor;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.LoggingRunnable;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.ServerServices;
+import org.apache.accumulo.core.util.ServerServices.Service;
+import org.apache.accumulo.core.util.SimpleThreadPool;
+import org.apache.accumulo.core.util.Stat;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockWatcher;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.client.ClientServiceHandler;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.conf.TableConfiguration;
+import org.apache.accumulo.server.data.ServerMutation;
+import org.apache.accumulo.server.logger.LogFileKey;
+import org.apache.accumulo.server.logger.LogFileValue;
+import org.apache.accumulo.server.master.state.Assignment;
+import org.apache.accumulo.server.master.state.DistributedStoreException;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletLocationState;
+import org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException;
+import org.apache.accumulo.server.master.state.TabletStateStore;
+import org.apache.accumulo.server.master.state.ZooTabletStateStore;
+import org.apache.accumulo.server.metrics.AbstractMetricsImpl;
+import org.apache.accumulo.server.problems.ProblemReport;
+import org.apache.accumulo.server.problems.ProblemReports;
+import org.apache.accumulo.server.security.AuditedSecurityOperation;
+import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.accumulo.server.security.SecurityOperation;
+import org.apache.accumulo.server.tabletserver.Compactor.CompactionInfo;
+import org.apache.accumulo.server.tabletserver.Tablet.CommitSession;
+import org.apache.accumulo.server.tabletserver.Tablet.KVEntry;
+import org.apache.accumulo.server.tabletserver.Tablet.LookupResult;
+import org.apache.accumulo.server.tabletserver.Tablet.MajorCompactionReason;
+import org.apache.accumulo.server.tabletserver.Tablet.MinorCompactionReason;
+import org.apache.accumulo.server.tabletserver.Tablet.ScanBatch;
+import org.apache.accumulo.server.tabletserver.Tablet.Scanner;
+import org.apache.accumulo.server.tabletserver.Tablet.SplitInfo;
+import org.apache.accumulo.server.tabletserver.Tablet.TConstraintViolationException;
+import org.apache.accumulo.server.tabletserver.Tablet.TabletClosedException;
+import org.apache.accumulo.server.tabletserver.TabletServerResourceManager.TabletResourceManager;
+import org.apache.accumulo.server.tabletserver.TabletStatsKeeper.Operation;
+import org.apache.accumulo.server.tabletserver.log.DfsLogger;
+import org.apache.accumulo.server.tabletserver.log.LogSorter;
+import org.apache.accumulo.server.tabletserver.log.MutationReceiver;
+import org.apache.accumulo.server.tabletserver.log.TabletServerLogger;
+import org.apache.accumulo.server.tabletserver.mastermessage.MasterMessage;
+import org.apache.accumulo.server.tabletserver.mastermessage.SplitReportMessage;
+import org.apache.accumulo.server.tabletserver.mastermessage.TabletStatusMessage;
+import org.apache.accumulo.server.tabletserver.metrics.TabletServerMBean;
+import org.apache.accumulo.server.tabletserver.metrics.TabletServerMinCMetrics;
+import org.apache.accumulo.server.tabletserver.metrics.TabletServerScanMetrics;
+import org.apache.accumulo.server.tabletserver.metrics.TabletServerUpdateMetrics;
+import org.apache.accumulo.server.trace.TraceFileSystem;
+import org.apache.accumulo.server.util.FileSystemMonitor;
+import org.apache.accumulo.server.util.Halt;
+import org.apache.accumulo.server.util.MapCounter;
+import org.apache.accumulo.server.util.MetadataTable;
+import org.apache.accumulo.server.util.MetadataTable.LogEntry;
+import org.apache.accumulo.server.util.TServerUtils;
+import org.apache.accumulo.server.util.TServerUtils.ServerPort;
+import org.apache.accumulo.server.util.time.RelativeTime;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
+import org.apache.accumulo.server.zookeeper.TransactionWatcher;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.start.Platform;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.accumulo.start.classloader.vfs.ContextManager;
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.instrument.Trace;
+import org.apache.accumulo.trace.instrument.thrift.TraceWrap;
+import org.apache.accumulo.trace.thrift.TInfo;
+import org.apache.commons.collections.map.LRUMap;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.Trash;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.SequenceFile.Reader;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+import org.apache.thrift.TProcessor;
+import org.apache.thrift.TServiceClient;
+import org.apache.thrift.server.TServer;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+
+enum ScanRunState {
+ QUEUED, RUNNING, FINISHED
+}
+
+public class TabletServer extends AbstractMetricsImpl implements org.apache.accumulo.server.tabletserver.metrics.TabletServerMBean {
+ private static final Logger log = Logger.getLogger(TabletServer.class);
+
+ private static HashMap<String,Long> prevGcTime = new HashMap<String,Long>();
+ private static long lastMemorySize = 0;
+ private static long gcTimeIncreasedCount;
+
+ private static final long MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS = 1000;
+
+ private TabletServerLogger logger;
+
+ protected TabletServerMinCMetrics mincMetrics = new TabletServerMinCMetrics();
+
+ private ServerConfiguration serverConfig;
+ private LogSorter logSorter = null;
+
+ public TabletServer(ServerConfiguration conf, FileSystem fs) {
+ super();
+ this.serverConfig = conf;
+ this.instance = conf.getInstance();
+ this.fs = TraceFileSystem.wrap(fs);
+ this.logSorter = new LogSorter(instance, fs, getSystemConfiguration());
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ synchronized (onlineTablets) {
+ long now = System.currentTimeMillis();
+ for (Tablet tablet : onlineTablets.values())
+ try {
+ tablet.updateRates(now);
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ }
+ }
+ }, 5000, 5000);
+ }
+
+ private synchronized static void logGCInfo(AccumuloConfiguration conf) {
+ List<GarbageCollectorMXBean> gcmBeans = ManagementFactory.getGarbageCollectorMXBeans();
+ Runtime rt = Runtime.getRuntime();
+
+ StringBuilder sb = new StringBuilder("gc");
+
+ boolean sawChange = false;
+
+ long maxIncreaseInCollectionTime = 0;
+
+ for (GarbageCollectorMXBean gcBean : gcmBeans) {
+ Long prevTime = prevGcTime.get(gcBean.getName());
+ long pt = 0;
+ if (prevTime != null) {
+ pt = prevTime;
+ }
+
+ long time = gcBean.getCollectionTime();
+
+ if (time - pt != 0) {
+ sawChange = true;
+ }
+
+ long increaseInCollectionTime = time - pt;
+ sb.append(String.format(" %s=%,.2f(+%,.2f) secs", gcBean.getName(), time / 1000.0, increaseInCollectionTime / 1000.0));
+ maxIncreaseInCollectionTime = Math.max(increaseInCollectionTime, maxIncreaseInCollectionTime);
+ prevGcTime.put(gcBean.getName(), time);
+ }
+
+ long mem = rt.freeMemory();
+ if (maxIncreaseInCollectionTime == 0) {
+ gcTimeIncreasedCount = 0;
+ } else {
+ gcTimeIncreasedCount++;
+ if (gcTimeIncreasedCount > 3 && mem < rt.maxMemory() * 0.05) {
+ log.warn("Running low on memory");
+ gcTimeIncreasedCount = 0;
+ }
+ }
+
+ if (mem > lastMemorySize) {
+ sawChange = true;
+ }
+
+ String sign = "+";
+ if (mem - lastMemorySize <= 0) {
+ sign = "";
+ }
+
+ sb.append(String.format(" freemem=%,d(%s%,d) totalmem=%,d", mem, sign, (mem - lastMemorySize), rt.totalMemory()));
+
+ if (sawChange) {
+ log.debug(sb.toString());
+ }
+
+ final long keepAliveTimeout = conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT);
+ if (maxIncreaseInCollectionTime > keepAliveTimeout) {
+ Halt.halt("Garbage collection may be interfering with lock keep-alive. Halting.", -1);
+ }
+
+ lastMemorySize = mem;
+ }
+
+ private TabletStatsKeeper statsKeeper;
+
+ private static class Session {
+ long lastAccessTime;
+ long startTime;
+ String user;
+ String client = TServerUtils.clientAddress.get();
+ public boolean reserved;
+
+ public void cleanup() {}
+ }
+
+ private static class SessionManager {
+
+ SecureRandom random;
+ Map<Long,Session> sessions;
+
+ SessionManager(AccumuloConfiguration conf) {
+ random = new SecureRandom();
+ sessions = new HashMap<Long,Session>();
+
+ final long maxIdle = conf.getTimeInMillis(Property.TSERV_SESSION_MAXIDLE);
+
+ Runnable r = new Runnable() {
+ @Override
+ public void run() {
+ sweep(maxIdle);
+ }
+ };
+
+ SimpleTimer.getInstance().schedule(r, 0, Math.max(maxIdle / 2, 1000));
+ }
+
+ synchronized long createSession(Session session, boolean reserve) {
+ long sid = random.nextLong();
+
+ while (sessions.containsKey(sid)) {
+ sid = random.nextLong();
+ }
+
+ sessions.put(sid, session);
+
+ session.reserved = reserve;
+
+ session.startTime = session.lastAccessTime = System.currentTimeMillis();
+
+ return sid;
+ }
+
+ /**
+ * while a session is reserved, it cannot be canceled or removed
+ *
+ * @param sessionId
+ */
+
+ synchronized Session reserveSession(long sessionId) {
+ Session session = sessions.get(sessionId);
+ if (session != null) {
+ if (session.reserved)
+ throw new IllegalStateException();
+ session.reserved = true;
+ }
+
+ return session;
+
+ }
+
+ synchronized void unreserveSession(Session session) {
+ if (!session.reserved)
+ throw new IllegalStateException();
+ session.reserved = false;
+ session.lastAccessTime = System.currentTimeMillis();
+ }
+
+ synchronized void unreserveSession(long sessionId) {
+ Session session = getSession(sessionId);
+ if (session != null)
+ unreserveSession(session);
+ }
+
+ synchronized Session getSession(long sessionId) {
+ Session session = sessions.get(sessionId);
+ if (session != null)
+ session.lastAccessTime = System.currentTimeMillis();
+ return session;
+ }
+
+ Session removeSession(long sessionId) {
+ Session session = null;
+ synchronized (this) {
+ session = sessions.remove(sessionId);
+ }
+
+ // do clean up out side of lock..
+ if (session != null)
+ session.cleanup();
+
+ return session;
+ }
+
+ private void sweep(long maxIdle) {
+ ArrayList<Session> sessionsToCleanup = new ArrayList<Session>();
+ synchronized (this) {
+ Iterator<Session> iter = sessions.values().iterator();
+ while (iter.hasNext()) {
+ Session session = iter.next();
+ long idleTime = System.currentTimeMillis() - session.lastAccessTime;
+ if (idleTime > maxIdle && !session.reserved) {
+ iter.remove();
+ sessionsToCleanup.add(session);
+ }
+ }
+ }
+
+ // do clean up outside of lock
+ for (Session session : sessionsToCleanup) {
+ session.cleanup();
+ }
+ }
+
+ synchronized void removeIfNotAccessed(final long sessionId, long delay) {
+ Session session = sessions.get(sessionId);
+ if (session != null) {
+ final long removeTime = session.lastAccessTime;
+ TimerTask r = new TimerTask() {
+ @Override
+ public void run() {
+ Session sessionToCleanup = null;
+ synchronized (SessionManager.this) {
+ Session session2 = sessions.get(sessionId);
+ if (session2 != null && session2.lastAccessTime == removeTime && !session2.reserved) {
+ sessions.remove(sessionId);
+ sessionToCleanup = session2;
+ }
+ }
+
+ // call clean up outside of lock
+ if (sessionToCleanup != null)
+ sessionToCleanup.cleanup();
+ }
+ };
+
+ SimpleTimer.getInstance().schedule(r, delay);
+ }
+ }
+
+ public synchronized Map<String,MapCounter<ScanRunState>> getActiveScansPerTable() {
+ Map<String,MapCounter<ScanRunState>> counts = new HashMap<String,MapCounter<ScanRunState>>();
+ for (Entry<Long,Session> entry : sessions.entrySet()) {
+
+ Session session = entry.getValue();
+ @SuppressWarnings("rawtypes")
+ ScanTask nbt = null;
+ String tableID = null;
+
+ if (session instanceof ScanSession) {
+ ScanSession ss = (ScanSession) session;
+ nbt = ss.nextBatchTask;
+ tableID = ss.extent.getTableId().toString();
+ } else if (session instanceof MultiScanSession) {
+ MultiScanSession mss = (MultiScanSession) session;
+ nbt = mss.lookupTask;
+ tableID = mss.threadPoolExtent.getTableId().toString();
+ }
+
+ if (nbt == null)
+ continue;
+
+ ScanRunState srs = nbt.getScanRunState();
+
+ if (srs == ScanRunState.FINISHED)
+ continue;
+
+ MapCounter<ScanRunState> stateCounts = counts.get(tableID);
+ if (stateCounts == null) {
+ stateCounts = new MapCounter<ScanRunState>();
+ counts.put(tableID, stateCounts);
+ }
+
+ stateCounts.increment(srs, 1);
+ }
+
+ return counts;
+ }
+
+ public synchronized List<ActiveScan> getActiveScans() {
+
+ ArrayList<ActiveScan> activeScans = new ArrayList<ActiveScan>();
+
+ long ct = System.currentTimeMillis();
+
+ for (Entry<Long,Session> entry : sessions.entrySet()) {
+ Session session = entry.getValue();
+ if (session instanceof ScanSession) {
+ ScanSession ss = (ScanSession) session;
+
+ ScanState state = ScanState.RUNNING;
+
+ ScanTask<ScanBatch> nbt = ss.nextBatchTask;
+ if (nbt == null) {
+ state = ScanState.IDLE;
+ } else {
+ switch (nbt.getScanRunState()) {
+ case QUEUED:
+ state = ScanState.QUEUED;
+ break;
+ case FINISHED:
+ state = ScanState.IDLE;
+ break;
+ case RUNNING:
+ default:
+ /* do nothing */
+ break;
+ }
+ }
+
+ activeScans.add(new ActiveScan(ss.client, ss.user, ss.extent.getTableId().toString(), ct - ss.startTime, ct - ss.lastAccessTime, ScanType.SINGLE,
+ state, ss.extent.toThrift(), Translator.translate(ss.columnSet, Translators.CT), ss.ssiList, ss.ssio, ss.auths.getAuthorizationsBB()));
+
+ } else if (session instanceof MultiScanSession) {
+ MultiScanSession mss = (MultiScanSession) session;
+
+ ScanState state = ScanState.RUNNING;
+
+ ScanTask<MultiScanResult> nbt = mss.lookupTask;
+ if (nbt == null) {
+ state = ScanState.IDLE;
+ } else {
+ switch (nbt.getScanRunState()) {
+ case QUEUED:
+ state = ScanState.QUEUED;
+ break;
+ case FINISHED:
+ state = ScanState.IDLE;
+ break;
+ case RUNNING:
+ default:
+ /* do nothing */
+ break;
+ }
+ }
+
+ activeScans.add(new ActiveScan(mss.client, mss.user, mss.threadPoolExtent.getTableId().toString(), ct - mss.startTime, ct - mss.lastAccessTime,
+ ScanType.BATCH, state, mss.threadPoolExtent.toThrift(), Translator.translate(mss.columnSet, Translators.CT), mss.ssiList, mss.ssio, mss.auths
+ .getAuthorizationsBB()));
+ }
+ }
+
+ return activeScans;
+ }
+ }
+
+ static class TservConstraintEnv implements Environment {
+
+ private TCredentials credentials;
+ private SecurityOperation security;
+ private Authorizations auths;
+ private KeyExtent ke;
+
+ TservConstraintEnv(SecurityOperation secOp, TCredentials credentials) {
+ this.security = secOp;
+ this.credentials = credentials;
+ }
+
+ void setExtent(KeyExtent ke) {
+ this.ke = ke;
+ }
+
+ @Override
+ public KeyExtent getExtent() {
+ return ke;
+ }
+
+ @Override
+ public String getUser() {
+ return credentials.getPrincipal();
+ }
+
+ @Override
+ public Authorizations getAuthorizations() {
+ if (auths == null)
+ try {
+ this.auths = security.getUserAuthorizations(credentials);
+ } catch (ThriftSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ return auths;
+ }
+
+ }
+
+ private abstract class ScanTask<T> implements RunnableFuture<T> {
+
+ protected AtomicBoolean interruptFlag;
+ protected ArrayBlockingQueue<Object> resultQueue;
+ protected AtomicInteger state;
+ protected AtomicReference<ScanRunState> runState;
+
+ private static final int INITIAL = 1;
+ private static final int ADDED = 2;
+ private static final int CANCELED = 3;
+
+ ScanTask() {
+ interruptFlag = new AtomicBoolean(false);
+ runState = new AtomicReference<ScanRunState>(ScanRunState.QUEUED);
+ state = new AtomicInteger(INITIAL);
+ resultQueue = new ArrayBlockingQueue<Object>(1);
+ }
+
+ protected void addResult(Object o) {
+ if (state.compareAndSet(INITIAL, ADDED))
+ resultQueue.add(o);
+ else if (state.get() == ADDED)
+ throw new IllegalStateException("Tried to add more than one result");
+ }
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ if (!mayInterruptIfRunning)
+ throw new IllegalArgumentException("Cancel will always attempt to interupt running next batch task");
+
+ if (state.get() == CANCELED)
+ return true;
+
+ if (state.compareAndSet(INITIAL, CANCELED)) {
+ interruptFlag.set(true);
+ resultQueue = null;
+ return true;
+ }
+
+ return false;
+ }
+
+ @Override
+ public T get() throws InterruptedException, ExecutionException {
+ throw new UnsupportedOperationException();
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
+
+ ArrayBlockingQueue<Object> localRQ = resultQueue;
+
+ if (state.get() == CANCELED)
+ throw new CancellationException();
+
+ if (localRQ == null && state.get() == ADDED)
+ throw new IllegalStateException("Tried to get result twice");
+
+ Object r = localRQ.poll(timeout, unit);
+
+ // could have been canceled while waiting
+ if (state.get() == CANCELED) {
+ if (r != null)
+ throw new IllegalStateException("Nothing should have been added when in canceled state");
+
+ throw new CancellationException();
+ }
+
+ if (r == null)
+ throw new TimeoutException();
+
+ // make this method stop working now that something is being
+ // returned
+ resultQueue = null;
+
+ if (r instanceof Throwable)
+ throw new ExecutionException((Throwable) r);
+
+ return (T) r;
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return state.get() == CANCELED;
+ }
+
+ @Override
+ public boolean isDone() {
+ return runState.get().equals(ScanRunState.FINISHED);
+ }
+
+ public ScanRunState getScanRunState() {
+ return runState.get();
+ }
+
+ }
+
+ private static class UpdateSession extends Session {
+ public Tablet currentTablet;
+ public MapCounter<Tablet> successfulCommits = new MapCounter<Tablet>();
+ Map<KeyExtent,Long> failures = new HashMap<KeyExtent,Long>();
+ HashMap<KeyExtent,SecurityErrorCode> authFailures = new HashMap<KeyExtent,SecurityErrorCode>();
+ public Violations violations;
+ public TCredentials credentials;
+ public long totalUpdates = 0;
+ public long flushTime = 0;
+ Stat prepareTimes = new Stat();
+ Stat walogTimes = new Stat();
+ Stat commitTimes = new Stat();
+ Stat authTimes = new Stat();
+ public Map<Tablet,List<Mutation>> queuedMutations = new HashMap<Tablet,List<Mutation>>();
+ public long queuedMutationSize = 0;
+ TservConstraintEnv cenv = null;
+ }
+
+ private static class ScanSession extends Session {
+ public KeyExtent extent;
+ public HashSet<Column> columnSet;
+ public List<IterInfo> ssiList;
+ public Map<String,Map<String,String>> ssio;
+ public Authorizations auths;
+ public long entriesReturned = 0;
+ public Stat nbTimes = new Stat();
+ public long batchCount = 0;
+ public volatile ScanTask<ScanBatch> nextBatchTask;
+ public AtomicBoolean interruptFlag;
+ public Scanner scanner;
+
+ @Override
+ public void cleanup() {
+ try {
+ if (nextBatchTask != null)
+ nextBatchTask.cancel(true);
+ } finally {
+ if (scanner != null)
+ scanner.close();
+ }
+ }
+
+ }
+
+ private static class MultiScanSession extends Session {
+ HashSet<Column> columnSet;
+ Map<KeyExtent,List<Range>> queries;
+ public List<IterInfo> ssiList;
+ public Map<String,Map<String,String>> ssio;
+ public Authorizations auths;
+
+ // stats
+ int numRanges;
+ int numTablets;
+ int numEntries;
+ long totalLookupTime;
+
+ public volatile ScanTask<MultiScanResult> lookupTask;
+ public KeyExtent threadPoolExtent;
+
+ @Override
+ public void cleanup() {
+ if (lookupTask != null)
+ lookupTask.cancel(true);
+ }
+ }
+
+ /**
+ * This little class keeps track of writes in progress and allows readers to wait for writes that started before the read. It assumes that the operation ids
+ * are monotonically increasing.
+ *
+ */
+ static class WriteTracker {
+ private static AtomicLong operationCounter = new AtomicLong(1);
+ private Map<TabletType,TreeSet<Long>> inProgressWrites = new EnumMap<TabletType,TreeSet<Long>>(TabletType.class);
+
+ WriteTracker() {
+ for (TabletType ttype : TabletType.values()) {
+ inProgressWrites.put(ttype, new TreeSet<Long>());
+ }
+ }
+
+ synchronized long startWrite(TabletType ttype) {
+ long operationId = operationCounter.getAndIncrement();
+ inProgressWrites.get(ttype).add(operationId);
+ return operationId;
+ }
+
+ synchronized void finishWrite(long operationId) {
+ if (operationId == -1)
+ return;
+
+ boolean removed = false;
+
+ for (TabletType ttype : TabletType.values()) {
+ removed = inProgressWrites.get(ttype).remove(operationId);
+ if (removed)
+ break;
+ }
+
+ if (!removed) {
+ throw new IllegalArgumentException("Attempted to finish write not in progress, operationId " + operationId);
+ }
+
+ this.notifyAll();
+ }
+
+ synchronized void waitForWrites(TabletType ttype) {
+ long operationId = operationCounter.getAndIncrement();
+ while (inProgressWrites.get(ttype).floor(operationId) != null) {
+ try {
+ this.wait();
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ }
+ }
+ }
+
+ public long startWrite(Set<Tablet> keySet) {
+ if (keySet.size() == 0)
+ return -1;
+
+ ArrayList<KeyExtent> extents = new ArrayList<KeyExtent>(keySet.size());
+
+ for (Tablet tablet : keySet)
+ extents.add(tablet.getExtent());
+
+ return startWrite(TabletType.type(extents));
+ }
+ }
+
+ public AccumuloConfiguration getSystemConfiguration() {
+ return serverConfig.getConfiguration();
+ }
+
+ TransactionWatcher watcher = new TransactionWatcher();
+
+ private class ThriftClientHandler extends ClientServiceHandler implements TabletClientService.Iface {
+
+ SessionManager sessionManager;
+
+ AccumuloConfiguration acuConf = getSystemConfiguration();
+
+ TabletServerUpdateMetrics updateMetrics = new TabletServerUpdateMetrics();
+
+ TabletServerScanMetrics scanMetrics = new TabletServerScanMetrics();
+
+ WriteTracker writeTracker = new WriteTracker();
+
+ ThriftClientHandler() {
+ super(instance, watcher);
+ log.debug(ThriftClientHandler.class.getName() + " created");
+ sessionManager = new SessionManager(getSystemConfiguration());
+ // Register the metrics MBean
+ try {
+ updateMetrics.register();
+ scanMetrics.register();
+ } catch (Exception e) {
+ log.error("Exception registering MBean with MBean Server", e);
+ }
+ }
+
+ @Override
+ public List<TKeyExtent> bulkImport(TInfo tinfo, TCredentials credentials, long tid, Map<TKeyExtent,Map<String,MapFileInfo>> files, boolean setTime)
+ throws ThriftSecurityException {
+
+ if (!security.canPerformSystemActions(credentials))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ List<TKeyExtent> failures = new ArrayList<TKeyExtent>();
+
+ for (Entry<TKeyExtent,Map<String,MapFileInfo>> entry : files.entrySet()) {
+ TKeyExtent tke = entry.getKey();
+ Map<String,MapFileInfo> fileMap = entry.getValue();
+
+ Tablet importTablet = onlineTablets.get(new KeyExtent(tke));
+
+ if (importTablet == null) {
+ failures.add(tke);
+ } else {
+ try {
+ importTablet.importMapFiles(tid, fileMap, setTime);
+ } catch (IOException ioe) {
+ log.info("files " + fileMap.keySet() + " not imported to " + new KeyExtent(tke) + ": " + ioe.getMessage());
+ failures.add(tke);
+ }
+ }
+ }
+ return failures;
+ }
+
+ private class NextBatchTask extends ScanTask<ScanBatch> {
+
+ private long scanID;
+
+ NextBatchTask(long scanID, AtomicBoolean interruptFlag) {
+ this.scanID = scanID;
+ this.interruptFlag = interruptFlag;
+
+ if (interruptFlag.get())
+ cancel(true);
+ }
+
+ @Override
+ public void run() {
+
+ final ScanSession scanSession = (ScanSession) sessionManager.getSession(scanID);
+ String oldThreadName = Thread.currentThread().getName();
+
+ try {
+ if (isCancelled() || scanSession == null)
+ return;
+
+ runState.set(ScanRunState.RUNNING);
+
+ Thread.currentThread().setName(
+ "User: " + scanSession.user + " Start: " + scanSession.startTime + " Client: " + scanSession.client + " Tablet: " + scanSession.extent);
+
+ Tablet tablet = onlineTablets.get(scanSession.extent);
+
+ if (tablet == null) {
+ addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
+ return;
+ }
+
+ long t1 = System.currentTimeMillis();
+ ScanBatch batch = scanSession.scanner.read();
+ long t2 = System.currentTimeMillis();
+ scanSession.nbTimes.addStat(t2 - t1);
+
+ // there should only be one thing on the queue at a time, so
+ // it should be ok to call add()
+ // instead of put()... if add() fails because queue is at
+ // capacity it means there is code
+ // problem somewhere
+ addResult(batch);
+ } catch (TabletClosedException e) {
+ addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
+ } catch (IterationInterruptedException iie) {
+ if (!isCancelled()) {
+ log.warn("Iteration interrupted, when scan not cancelled", iie);
+ addResult(iie);
+ }
+ } catch (TooManyFilesException tmfe) {
+ addResult(tmfe);
+ } catch (Throwable e) {
+ log.warn("exception while scanning tablet " + (scanSession == null ? "(unknown)" : scanSession.extent), e);
+ addResult(e);
+ } finally {
+ runState.set(ScanRunState.FINISHED);
+ Thread.currentThread().setName(oldThreadName);
+ }
+
+ }
+ }
+
+ private class LookupTask extends ScanTask<MultiScanResult> {
+
+ private long scanID;
+
+ LookupTask(long scanID) {
+ this.scanID = scanID;
+ }
+
+ @Override
+ public void run() {
+ MultiScanSession session = (MultiScanSession) sessionManager.getSession(scanID);
+ String oldThreadName = Thread.currentThread().getName();
+
+ try {
+ if (isCancelled() || session == null)
+ return;
+
+ TableConfiguration acuTableConf = ServerConfiguration.getTableConfiguration(instance, session.threadPoolExtent.getTableId().toString());
+ long maxResultsSize = acuTableConf.getMemoryInBytes(Property.TABLE_SCAN_MAXMEM);
+
+ runState.set(ScanRunState.RUNNING);
+ Thread.currentThread().setName("Client: " + session.client + " User: " + session.user + " Start: " + session.startTime + " Table: ");
+
+ long bytesAdded = 0;
+ long maxScanTime = 4000;
+
+ long startTime = System.currentTimeMillis();
+
+ ArrayList<KVEntry> results = new ArrayList<KVEntry>();
+ Map<KeyExtent,List<Range>> failures = new HashMap<KeyExtent,List<Range>>();
+ ArrayList<KeyExtent> fullScans = new ArrayList<KeyExtent>();
+ KeyExtent partScan = null;
+ Key partNextKey = null;
+ boolean partNextKeyInclusive = false;
+
+ Iterator<Entry<KeyExtent,List<Range>>> iter = session.queries.entrySet().iterator();
+
+ // check the time so that the read ahead thread is not monopolized
+ while (iter.hasNext() && bytesAdded < maxResultsSize && (System.currentTimeMillis() - startTime) < maxScanTime) {
+ Entry<KeyExtent,List<Range>> entry = iter.next();
+
+ iter.remove();
+
+ // check that tablet server is serving requested tablet
+ Tablet tablet = onlineTablets.get(entry.getKey());
+ if (tablet == null) {
+ failures.put(entry.getKey(), entry.getValue());
+ continue;
+ }
+ Thread.currentThread().setName(
+ "Client: " + session.client + " User: " + session.user + " Start: " + session.startTime + " Tablet: " + entry.getKey().toString());
+
+ LookupResult lookupResult;
+ try {
+
+ // do the following check to avoid a race condition
+ // between setting false below and the task being
+ // canceled
+ if (isCancelled())
+ interruptFlag.set(true);
+
+ lookupResult = tablet.lookup(entry.getValue(), session.columnSet, session.auths, results, maxResultsSize - bytesAdded, session.ssiList,
+ session.ssio, interruptFlag);
+
+ // if the tablet was closed it it possible that the
+ // interrupt flag was set.... do not want it set for
+ // the next
+ // lookup
+ interruptFlag.set(false);
+
+ } catch (IOException e) {
+ log.warn("lookup failed for tablet " + entry.getKey(), e);
+ throw new RuntimeException(e);
+ }
+
+ bytesAdded += lookupResult.bytesAdded;
+
+ if (lookupResult.unfinishedRanges.size() > 0) {
+ if (lookupResult.closed) {
+ failures.put(entry.getKey(), lookupResult.unfinishedRanges);
+ } else {
+ session.queries.put(entry.getKey(), lookupResult.unfinishedRanges);
+ partScan = entry.getKey();
+ partNextKey = lookupResult.unfinishedRanges.get(0).getStartKey();
+ partNextKeyInclusive = lookupResult.unfinishedRanges.get(0).isStartKeyInclusive();
+ }
+ } else {
+ fullScans.add(entry.getKey());
+ }
+ }
+
+ long finishTime = System.currentTimeMillis();
+ session.totalLookupTime += (finishTime - startTime);
+ session.numEntries += results.size();
+
+ // convert everything to thrift before adding result
+ List<TKeyValue> retResults = new ArrayList<TKeyValue>();
+ for (KVEntry entry : results)
+ retResults.add(new TKeyValue(entry.key.toThrift(), ByteBuffer.wrap(entry.value)));
+ Map<TKeyExtent,List<TRange>> retFailures = Translator.translate(failures, Translators.KET, new Translator.ListTranslator<Range,TRange>(Translators.RT));
+ List<TKeyExtent> retFullScans = Translator.translate(fullScans, Translators.KET);
+ TKeyExtent retPartScan = null;
+ TKey retPartNextKey = null;
+ if (partScan != null) {
+ retPartScan = partScan.toThrift();
+ retPartNextKey = partNextKey.toThrift();
+ }
+ // add results to queue
+ addResult(new MultiScanResult(retResults, retFailures, retFullScans, retPartScan, retPartNextKey, partNextKeyInclusive, session.queries.size() != 0));
+ } catch (IterationInterruptedException iie) {
+ if (!isCancelled()) {
+ log.warn("Iteration interrupted, when scan not cancelled", iie);
+ addResult(iie);
+ }
+ } catch (Throwable e) {
+ log.warn("exception while doing multi-scan ", e);
+ addResult(e);
+ } finally {
+ Thread.currentThread().setName(oldThreadName);
+ runState.set(ScanRunState.FINISHED);
+ }
+ }
+ }
+
+ @Override
+ public InitialScan startScan(TInfo tinfo, TCredentials credentials, TKeyExtent textent, TRange range, List<TColumn> columns, int batchSize,
+ List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites, boolean isolated)
+ throws NotServingTabletException, ThriftSecurityException, org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
+
+ Authorizations userauths = null;
+ if (!security.canScan(credentials, new String(textent.getTable(), Constants.UTF8)))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ userauths = security.getUserAuthorizations(credentials);
+ for (ByteBuffer auth : authorizations)
+ if (!userauths.contains(ByteBufferUtil.toBytes(auth)))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
+
+ KeyExtent extent = new KeyExtent(textent);
+
+ // wait for any writes that are in flight.. this done to ensure
+ // consistency across client restarts... assume a client writes
+ // to accumulo and dies while waiting for a confirmation from
+ // accumulo... the client process restarts and tries to read
+ // data from accumulo making the assumption that it will get
+ // any writes previously made... however if the server side thread
+ // processing the write from the dead client is still in progress,
+ // the restarted client may not see the write unless we wait here.
+ // this behavior is very important when the client is reading the
+ // !METADATA table
+ if (waitForWrites)
+ writeTracker.waitForWrites(TabletType.type(extent));
+
+ Tablet tablet = onlineTablets.get(extent);
+ if (tablet == null)
+ throw new NotServingTabletException(textent);
+
+ ScanSession scanSession = new ScanSession();
+ scanSession.user = credentials.getPrincipal();
+ scanSession.extent = new KeyExtent(extent);
+ scanSession.columnSet = new HashSet<Column>();
+ scanSession.ssiList = ssiList;
+ scanSession.ssio = ssio;
+ scanSession.auths = new Authorizations(authorizations);
+ scanSession.interruptFlag = new AtomicBoolean();
+
+ for (TColumn tcolumn : columns) {
+ scanSession.columnSet.add(new Column(tcolumn));
+ }
+
+ scanSession.scanner = tablet.createScanner(new Range(range), batchSize, scanSession.columnSet, scanSession.auths, ssiList, ssio, isolated,
+ scanSession.interruptFlag);
+
+ long sid = sessionManager.createSession(scanSession, true);
+
+ ScanResult scanResult;
+ try {
+ scanResult = continueScan(tinfo, sid, scanSession);
+ } catch (NoSuchScanIDException e) {
+ log.error("The impossible happened", e);
+ throw new RuntimeException();
+ } finally {
+ sessionManager.unreserveSession(sid);
+ }
+
+ return new InitialScan(sid, scanResult);
+ }
+
+ @Override
+ public ScanResult continueScan(TInfo tinfo, long scanID) throws NoSuchScanIDException, NotServingTabletException,
+ org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
+ ScanSession scanSession = (ScanSession) sessionManager.reserveSession(scanID);
+ if (scanSession == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ try {
+ return continueScan(tinfo, scanID, scanSession);
+ } finally {
+ sessionManager.unreserveSession(scanSession);
+ }
+ }
+
+ private ScanResult continueScan(TInfo tinfo, long scanID, ScanSession scanSession) throws NoSuchScanIDException, NotServingTabletException,
+ org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
+
+ if (scanSession.nextBatchTask == null) {
+ scanSession.nextBatchTask = new NextBatchTask(scanID, scanSession.interruptFlag);
+ resourceManager.executeReadAhead(scanSession.extent, scanSession.nextBatchTask);
+ }
+
+ ScanBatch bresult;
+ try {
+ bresult = scanSession.nextBatchTask.get(MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS, TimeUnit.MILLISECONDS);
+ scanSession.nextBatchTask = null;
+ } catch (ExecutionException e) {
+ sessionManager.removeSession(scanID);
+ if (e.getCause() instanceof NotServingTabletException)
+ throw (NotServingTabletException) e.getCause();
+ else if (e.getCause() instanceof TooManyFilesException)
+ throw new org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException(scanSession.extent.toThrift());
+ else
+ throw new RuntimeException(e);
+ } catch (CancellationException ce) {
+ sessionManager.removeSession(scanID);
+ Tablet tablet = onlineTablets.get(scanSession.extent);
+ if (tablet == null || tablet.isClosed())
+ throw new NotServingTabletException(scanSession.extent.toThrift());
+ else
+ throw new NoSuchScanIDException();
+ } catch (TimeoutException e) {
+ List<TKeyValue> param = Collections.emptyList();
+ long timeout = acuConf.getTimeInMillis(Property.TSERV_CLIENT_TIMEOUT);
+ sessionManager.removeIfNotAccessed(scanID, timeout);
+ return new ScanResult(param, true);
+ } catch (Throwable t) {
+ sessionManager.removeSession(scanID);
+ log.warn("Failed to get next batch", t);
+ throw new RuntimeException(t);
+ }
+
+ ScanResult scanResult = new ScanResult(Key.compress(bresult.results), bresult.more);
+
+ scanSession.entriesReturned += scanResult.results.size();
+
+ scanSession.batchCount++;
+
+ if (scanResult.more && scanSession.batchCount > 3) {
+ // start reading next batch while current batch is transmitted
+ // to client
+ scanSession.nextBatchTask = new NextBatchTask(scanID, scanSession.interruptFlag);
+ resourceManager.executeReadAhead(scanSession.extent, scanSession.nextBatchTask);
+ }
+
+ if (!scanResult.more)
+ closeScan(tinfo, scanID);
+
+ return scanResult;
+ }
+
+ @Override
+ public void closeScan(TInfo tinfo, long scanID) {
+ ScanSession ss = (ScanSession) sessionManager.removeSession(scanID);
+ if (ss != null) {
+ long t2 = System.currentTimeMillis();
+
+ log.debug(String.format("ScanSess tid %s %s %,d entries in %.2f secs, nbTimes = [%s] ", TServerUtils.clientAddress.get(), ss.extent.getTableId()
+ .toString(), ss.entriesReturned, (t2 - ss.startTime) / 1000.0, ss.nbTimes.toString()));
+ if (scanMetrics.isEnabled()) {
+ scanMetrics.add(TabletServerScanMetrics.scan, t2 - ss.startTime);
+ scanMetrics.add(TabletServerScanMetrics.resultSize, ss.entriesReturned);
+ }
+ }
+ }
+
+ @Override
+ public InitialMultiScan startMultiScan(TInfo tinfo, TCredentials credentials, Map<TKeyExtent,List<TRange>> tbatch, List<TColumn> tcolumns,
+ List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites) throws ThriftSecurityException {
+ // find all of the tables that need to be scanned
+ HashSet<String> tables = new HashSet<String>();
+ for (TKeyExtent keyExtent : tbatch.keySet()) {
+ tables.add(new String(keyExtent.getTable(), Constants.UTF8));
+ }
+
+ // check if user has permission to the tables
+ Authorizations userauths = null;
+ for (String table : tables)
+ if (!security.canScan(credentials, table))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ userauths = security.getUserAuthorizations(credentials);
+ for (ByteBuffer auth : authorizations)
+ if (!userauths.contains(ByteBufferUtil.toBytes(auth)))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
+
+ KeyExtent threadPoolExtent = null;
+
+ Map<KeyExtent,List<Range>> batch = Translator.translate(tbatch, Translators.TKET, new Translator.ListTranslator<TRange,Range>(Translators.TRT));
+
+ for (KeyExtent keyExtent : batch.keySet()) {
+ if (threadPoolExtent == null) {
+ threadPoolExtent = keyExtent;
+ } else if (keyExtent.isRootTablet()) {
+ throw new IllegalArgumentException("Cannot batch query root tablet with other tablets " + threadPoolExtent + " " + keyExtent);
+ } else if (keyExtent.isMeta() && !threadPoolExtent.isMeta()) {
+ throw new IllegalArgumentException("Cannot batch query !METADATA and non !METADATA tablets " + threadPoolExtent + " " + keyExtent);
+ }
+
+ }
+
+ if (waitForWrites)
+ writeTracker.waitForWrites(TabletType.type(batch.keySet()));
+
+ MultiScanSession mss = new MultiScanSession();
+ mss.user = credentials.getPrincipal();
+ mss.queries = batch;
+ mss.columnSet = new HashSet<Column>(tcolumns.size());
+ mss.ssiList = ssiList;
+ mss.ssio = ssio;
+ mss.auths = new Authorizations(authorizations);
+
+ mss.numTablets = batch.size();
+ for (List<Range> ranges : batch.values()) {
+ mss.numRanges += ranges.size();
+ }
+
+ for (TColumn tcolumn : tcolumns)
+ mss.columnSet.add(new Column(tcolumn));
+
+ mss.threadPoolExtent = threadPoolExtent;
+
+ long sid = sessionManager.createSession(mss, true);
+
+ MultiScanResult result;
+ try {
+ result = continueMultiScan(tinfo, sid, mss);
+ } catch (NoSuchScanIDException e) {
+ log.error("the impossible happened", e);
+ throw new RuntimeException("the impossible happened", e);
+ } finally {
+ sessionManager.unreserveSession(sid);
+ }
+
+ return new InitialMultiScan(sid, result);
+ }
+
+ @Override
+ public MultiScanResult continueMultiScan(TInfo tinfo, long scanID) throws NoSuchScanIDException {
+
+ MultiScanSession session = (MultiScanSession) sessionManager.reserveSession(scanID);
+
+ if (session == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ try {
+ return continueMultiScan(tinfo, scanID, session);
+ } finally {
+ sessionManager.unreserveSession(session);
+ }
+ }
+
+ private MultiScanResult continueMultiScan(TInfo tinfo, long scanID, MultiScanSession session) throws NoSuchScanIDException {
+
+ if (session.lookupTask == null) {
+ session.lookupTask = new LookupTask(scanID);
+ resourceManager.executeReadAhead(session.threadPoolExtent, session.lookupTask);
+ }
+
+ try {
+ MultiScanResult scanResult = session.lookupTask.get(MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS, TimeUnit.MILLISECONDS);
+ session.lookupTask = null;
+ return scanResult;
+ } catch (TimeoutException e1) {
+ long timeout = acuConf.getTimeInMillis(Property.TSERV_CLIENT_TIMEOUT);
+ sessionManager.removeIfNotAccessed(scanID, timeout);
+ List<TKeyValue> results = Collections.emptyList();
+ Map<TKeyExtent,List<TRange>> failures = Collections.emptyMap();
+ List<TKeyExtent> fullScans = Collections.emptyList();
+ return new MultiScanResult(results, failures, fullScans, null, null, false, true);
+ } catch (Throwable t) {
+ sessionManager.removeSession(scanID);
+ log.warn("Failed to get multiscan result", t);
+ throw new RuntimeException(t);
+ }
+ }
+
+ @Override
+ public void closeMultiScan(TInfo tinfo, long scanID) throws NoSuchScanIDException {
+ MultiScanSession session = (MultiScanSession) sessionManager.removeSession(scanID);
+ if (session == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ long t2 = System.currentTimeMillis();
+ log.debug(String.format("MultiScanSess %s %,d entries in %.2f secs (lookup_time:%.2f secs tablets:%,d ranges:%,d) ", TServerUtils.clientAddress.get(),
+ session.numEntries, (t2 - session.startTime) / 1000.0, session.totalLookupTime / 1000.0, session.numTablets, session.numRanges));
+ }
+
+ @Override
+ public long startUpdate(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException {
+ // Make sure user is real
+
+ security.authenticateUser(credentials, credentials);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
+
+ UpdateSession us = new UpdateSession();
+ us.violations = new Violations();
+ us.credentials = credentials;
+ us.cenv = new TservConstraintEnv(security, us.credentials);
+
+ long sid = sessionManager.createSession(us, false);
+
+ return sid;
+ }
+
+ private void setUpdateTablet(UpdateSession us, KeyExtent keyExtent) {
+ long t1 = System.currentTimeMillis();
+ if (us.currentTablet != null && us.currentTablet.getExtent().equals(keyExtent))
+ return;
+ if (us.currentTablet == null && (us.failures.containsKey(keyExtent) || us.authFailures.containsKey(keyExtent))) {
+ // if there were previous failures, then do not accept additional writes
+ return;
+ }
+
+ try {
+ // if user has no permission to write to this table, add it to
+ // the failures list
+ boolean sameTable = us.currentTablet != null && (us.currentTablet.getExtent().getTableId().equals(keyExtent.getTableId()));
+ if (sameTable || security.canWrite(us.credentials, keyExtent.getTableId().toString())) {
+ long t2 = System.currentTimeMillis();
+ us.authTimes.addStat(t2 - t1);
+ us.currentTablet = onlineTablets.get(keyExtent);
+ if (us.currentTablet != null) {
+ us.queuedMutations.put(us.currentTablet, new ArrayList<Mutation>());
+ } else {
+ // not serving tablet, so report all mutations as
+ // failures
+ us.failures.put(keyExtent, 0l);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.unknownTabletErrors, 0);
+ }
+ } else {
+ log.warn("Denying access to table " + keyExtent.getTableId() + " for user " + us.credentials.getPrincipal());
+ long t2 = System.currentTimeMillis();
+ us.authTimes.addStat(t2 - t1);
+ us.currentTablet = null;
+ us.authFailures.put(keyExtent, SecurityErrorCode.PERMISSION_DENIED);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
+ return;
+ }
+ } catch (ThriftSecurityException e) {
+ log.error("Denying permission to check user " + us.credentials.getPrincipal() + " with user " + e.getUser(), e);
+ long t2 = System.currentTimeMillis();
+ us.authTimes.addStat(t2 - t1);
+ us.currentTablet = null;
+ us.authFailures.put(keyExtent, e.getCode());
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
+ return;
+ }
+ }
+
+ @Override
+ public void applyUpdates(TInfo tinfo, long updateID, TKeyExtent tkeyExtent, List<TMutation> tmutations) {
+ UpdateSession us = (UpdateSession) sessionManager.reserveSession(updateID);
+ if (us == null) {
+ throw new RuntimeException("No Such SessionID");
+ }
+
+ try {
+ KeyExtent keyExtent = new KeyExtent(tkeyExtent);
+ setUpdateTablet(us, keyExtent);
+
+ if (us.currentTablet != null) {
+ List<Mutation> mutations = us.queuedMutations.get(us.currentTablet);
+ for (TMutation tmutation : tmutations) {
+ Mutation mutation = new ServerMutation(tmutation);
+ mutations.add(mutation);
+ us.queuedMutationSize += mutation.numBytes();
+ }
+ if (us.queuedMutationSize > getSystemConfiguration().getMemoryInBytes(Property.TSERV_MUTATION_QUEUE_MAX))
+ flush(us);
+ }
+ } finally {
+ sessionManager.unreserveSession(us);
+ }
+ }
+
+ private void flush(UpdateSession us) {
+
+ int mutationCount = 0;
+ Map<CommitSession,List<Mutation>> sendables = new HashMap<CommitSession,List<Mutation>>();
+ Throwable error = null;
+
+ long pt1 = System.currentTimeMillis();
+
+ boolean containsMetadataTablet = false;
+ for (Tablet tablet : us.queuedMutations.keySet())
+ if (tablet.getExtent().isMeta())
+ containsMetadataTablet = true;
+
+ if (!containsMetadataTablet && us.queuedMutations.size() > 0)
+ TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
+
+ Span prep = Trace.start("prep");
+ for (Entry<Tablet,? extends List<Mutation>> entry : us.queuedMutations.entrySet()) {
+
+ Tablet tablet = entry.getKey();
+ List<Mutation> mutations = entry.getValue();
+ if (mutations.size() > 0) {
+ try {
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.mutationArraySize, mutations.size());
+
+ CommitSession commitSession = tablet.prepareMutationsForCommit(us.cenv, mutations);
+ if (commitSession == null) {
+ if (us.currentTablet == tablet) {
+ us.currentTablet = null;
+ }
+ us.failures.put(tablet.getExtent(), us.successfulCommits.get(tablet));
+ } else {
+ sendables.put(commitSession, mutations);
+ mutationCount += mutations.size();
+ }
+
+ } catch (TConstraintViolationException e) {
+ us.violations.add(e.getViolations());
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.constraintViolations, 0);
+
+ if (e.getNonViolators().size() > 0) {
+ // only log and commit mutations if there were some
+ // that did not
+ // violate constraints... this is what
+ // prepareMutationsForCommit()
+ // expects
+ sendables.put(e.getCommitSession(), e.getNonViolators());
+ }
+
+ mutationCount += mutations.size();
+
+ } catch (HoldTimeoutException t) {
+ error = t;
+ log.debug("Giving up on mutations due to a long memory hold time");
+ break;
+ } catch (Throwable t) {
+ error = t;
+ log.error("Unexpected error preparing for commit", error);
+ break;
+ }
+ }
+ }
+ prep.stop();
+
+ Span wal = Trace.start("wal");
+ long pt2 = System.currentTimeMillis();
+ long avgPrepareTime = (long) ((pt2 - pt1) / (double) us.queuedMutations.size());
+ us.prepareTimes.addStat(pt2 - pt1);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.commitPrep, (avgPrepareTime));
+
+ if (error != null) {
+ for (Entry<CommitSession,List<Mutation>> e : sendables.entrySet()) {
+ e.getKey().abortCommit(e.getValue());
+ }
+ throw new RuntimeException(error);
+ }
+ try {
+ while (true) {
+ try {
+ long t1 = System.currentTimeMillis();
+
+ logger.logManyTablets(sendables);
+
+ long t2 = System.currentTimeMillis();
+ us.walogTimes.addStat(t2 - t1);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.waLogWriteTime, (t2 - t1));
+
+ break;
+ } catch (IOException ex) {
+ log.warn("logging mutations failed, retrying");
+ } catch (Throwable t) {
+ log.error("Unknown exception logging mutations, counts for mutations in flight not decremented!", t);
+ throw new RuntimeException(t);
+ }
+ }
+
+ wal.stop();
+
+ Span commit = Trace.start("commit");
+ long t1 = System.currentTimeMillis();
+ for (Entry<CommitSession,? extends List<Mutation>> entry : sendables.entrySet()) {
+ CommitSession commitSession = entry.getKey();
+ List<Mutation> mutations = entry.getValue();
+
+ commitSession.commit(mutations);
+
+ Tablet tablet = commitSession.getTablet();
+
+ if (tablet == us.currentTablet) {
+ // because constraint violations may filter out some
+ // mutations, for proper
+ // accounting with the client code, need to increment
+ // the count based
+ // on the original number of mutations from the client
+ // NOT the filtered number
+ us.successfulCommits.increment(tablet, us.queuedMutations.get(tablet).size());
+ }
+ }
+ long t2 = System.currentTimeMillis();
+
+ long avgCommitTime = (long) ((t2 - t1) / (double) sendables.size());
+
+ us.flushTime += (t2 - pt1);
+ us.commitTimes.addStat(t2 - t1);
+
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.commitTime, avgCommitTime);
+ commit.stop();
+ } finally {
+ us.queuedMutations.clear();
+ if (us.currentTablet != null) {
+ us.queuedMutations.put(us.currentTablet, new ArrayList<Mutation>());
+ }
+ us.queuedMutationSize = 0;
+ }
+ us.totalUpdates += mutationCount;
+ }
+
+ @Override
+ public UpdateErrors closeUpdate(TInfo tinfo, long updateID) throws NoSuchScanIDException {
+ UpdateSession us = (UpdateSession) sessionManager.removeSession(updateID);
+ if (us == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ // clients may or may not see data from an update session while
+ // it is in progress, however when the update session is closed
+ // want to ensure that reads wait for the write to finish
+ long opid = writeTracker.startWrite(us.queuedMutations.keySet());
+
+ try {
+ flush(us);
+ } finally {
+ writeTracker.finishWrite(opid);
+ }
+
+ log.debug(String.format("UpSess %s %,d in %.3fs, at=[%s] ft=%.3fs(pt=%.3fs lt=%.3fs ct=%.3fs)", TServerUtils.clientAddress.get(), us.totalUpdates,
+ (System.currentTimeMillis() - us.startTime) / 1000.0, us.authTimes.toString(), us.flushTime / 1000.0, us.prepareTimes.getSum() / 1000.0,
+ us.walogTimes.getSum() / 1000.0, us.commitTimes.getSum() / 1000.0));
+ if (us.failures.size() > 0) {
+ Entry<KeyExtent,Long> first = us.failures.entrySet().iterator().next();
+ log.debug(String.format("Failures: %d, first extent %s successful commits: %d", us.failures.size(), first.getKey().toString(), first.getValue()));
+ }
+ List<ConstraintViolationSummary> violations = us.violations.asList();
+ if (violations.size() > 0) {
+ ConstraintViolationSummary first = us.violations.asList().iterator().next();
+ log.debug(String.format("Violations: %d, first %s occurs %d", violations.size(), first.violationDescription, first.numberOfViolatingMutations));
+ }
+ if (us.authFailures.size() > 0) {
+ KeyExtent first = us.authFailures.keySet().iterator().next();
+ log.debug(String.format("Authentication Failures: %d, first %s", us.authFailures.size(), first.toString()));
+ }
+
+ return new UpdateErrors(Translator.translate(us.failures, Translators.KET), Translator.translate(violations, Translators.CVST), Translator.translate(
+ us.authFailures, Translators.KET));
+ }
+
+ @Override
+ public void update(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, TMutation tmutation) throws NotServingTabletException,
+ ConstraintViolationException, ThriftSecurityException {
+
+ if (!security.canWrite(credentials, new String(tkeyExtent.getTable(), Constants.UTF8)))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ KeyExtent keyExtent = new KeyExtent(tkeyExtent);
+ Tablet tablet = onlineTablets.get(new KeyExtent(keyExtent));
+ if (tablet == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+
+ if (!keyExtent.isMeta())
+ TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
+
+ long opid = writeTracker.startWrite(TabletType.type(keyExtent));
+
+ try {
+ Mutation mutation = new ServerMutation(tmutation);
+ List<Mutation> mutations = Collections.singletonList(mutation);
+
+ Span prep = Trace.start("prep");
+ CommitSession cs = tablet.prepareMutationsForCommit(new TservConstraintEnv(security, credentials), mutations);
+ prep.stop();
+ if (cs == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+
+ while (true) {
+ try {
+ Span wal = Trace.start("wal");
+ logger.log(cs, cs.getWALogSeq(), mutation);
+ wal.stop();
+ break;
+ } catch (IOException ex) {
+ log.warn(ex, ex);
+ }
+ }
+
+ Span commit = Trace.start("commit");
+ cs.commit(mutations);
+ commit.stop();
+ } catch (TConstraintViolationException e) {
+ throw new ConstraintViolationException(Translator.translate(e.getViolations().asList(), Translators.CVST));
+ } finally {
+ writeTracker.finishWrite(opid);
+ }
+ }
+
+ @Override
+ public void splitTablet(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, ByteBuffer splitPoint)
+ throws NotServingTabletException, ThriftSecurityException {
+
+ String tableId = new String(ByteBufferUtil.toBytes(tkeyExtent.table), Constants.UTF8);
+ if (!security.canSplitTablet(credentials, tableId))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ KeyExtent keyExtent = new KeyExtent(tkeyExtent);
+
+ Tablet tablet = onlineTablets.get(keyExtent);
+ if (tablet == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+
+ if (keyExtent.getEndRow() == null || !keyExtent.getEndRow().equals(ByteBufferUtil.toText(splitPoint))) {
+ try {
+ if (TabletServer.this.splitTablet(tablet, ByteBufferUtil.toBytes(splitPoint)) == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+ } catch (IOException e) {
+ log.warn("Failed to split " + keyExtent, e);
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Override
+ public TabletServerStatus getTabletServerStatus(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ return getStats(sessionManager.getActiveScansPerTable());
+ }
+
+ @Override
+ public List<TabletStats> getTabletStats(TInfo tinfo, TCredentials credentials, String tableId) throws ThriftSecurityException, TException {
+ TreeMap<KeyExtent,Tablet> onlineTabletsCopy;
+ synchronized (onlineTablets) {
+ onlineTabletsCopy = new TreeMap<KeyExtent,Tablet>(onlineTablets);
+ }
+ List<TabletStats> result = new ArrayList<TabletStats>();
+ Text text = new Text(tableId);
+ KeyExtent start = new KeyExtent(text, new Text(), null);
+ for (Entry<KeyExtent,Tablet> entry : onlineTabletsCopy.tailMap(start).entrySet()) {
+ KeyExtent ke = entry.getKey();
+ if (ke.getTableId().compareTo(text) == 0) {
+ Tablet tablet = entry.getValue();
+ TabletStats stats = tablet.timer.getTabletStats();
+ stats.extent = ke.toThrift();
+ stats.ingestRate = tablet.ingestRate();
+ stats.queryRate = tablet.queryRate();
+ stats.splitCreationTime = tablet.getSplitCreationTime();
+ stats.numEntries = tablet.getNumEntries();
+ result.add(stats);
+ }
+ }
+ return result;
+ }
+
+ private ZooCache masterLockCache = new ZooCache();
+
+ private void checkPermission(TCredentials credentials, String lock, boolean requiresSystemPermission, final String request)
+ throws ThriftSecurityException {
+ if (requiresSystemPermission) {
+ boolean fatal = false;
+ try {
+ log.debug("Got " + request + " message from user: " + credentials.getPrincipal());
+ if (!security.canPerformSystemActions(credentials)) {
+ log.warn("Got " + request + " message from user: " + credentials.getPrincipal());
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ }
+ } catch (ThriftSecurityException e) {
+ log.warn("Got " + request + " message from unauthenticatable user: " + e.getUser());
+ if (e.getUser().equals(SecurityConstants.SYSTEM_PRINCIPAL)) {
+ log.fatal("Got message from a service with a mismatched configuration. Please ensure a compatible configuration.", e);
+ fatal = true;
+ }
+ throw e;
+ } finally {
+ if (fatal) {
+ Halt.halt(1, new Runnable() {
+ @Override
+ public void run() {
+ logGCInfo(getSystemConfiguration());
+ }
+ });
+ }
+ }
+ }
+
+ if (tabletServerLock == null || !tabletServerLock.wasLockAcquired()) {
+ log.warn("Got " + request + " message from master before lock acquired, ignoring...");
+ throw new RuntimeException("Lock not acquired");
+ }
+
+ if (tabletServerLock != null && tabletServerLock.wasLockAcquired() && !tabletServerLock.isLocked()) {
+ Halt.halt(1, new Runnable() {
+ @Override
+ public void run() {
+ log.info("Tablet server no longer holds lock during checkPermission() : " + request + ", exiting");
+ logGCInfo(getSystemConfiguration());
+ }
+ });
+ }
+
+ if (lock != null) {
+ ZooUtil.LockID lid = new ZooUtil.LockID(ZooUtil.getRoot(instance) + Constants.ZMASTER_LOCK, lock);
+
+ try {
+ if (!ZooLock.isLockHeld(masterLockCache, lid)) {
+ // maybe the cache is out of date and a new master holds the
+ // lock?
+ masterLockCache.clear();
+ if (!ZooLock.isLockHeld(masterLockCache, lid)) {
+ log.warn("Got " + request + " message from a master that does not hold the current lock " + lock);
+ throw new RuntimeException("bad master lock");
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("bad master lock", e);
+ }
+ }
+ }
+
+ @Override
+ public void loadTablet(TInfo tinfo, TCredentials credentials, String lock, final TKeyExtent textent) {
+
+ try {
+ checkPermission(credentials, lock, true, "loadTablet");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+
+ final KeyExtent extent = new KeyExtent(textent);
+
+ synchronized (unopenedTablets) {
+ synchronized (openingTablets) {
+ synchronized (onlineTablets) {
+
+ // checking if this exact tablet is in any of the sets
+ // below is not a strong enough check
+ // when splits and fix splits occurring
+
+ Set<KeyExtent> unopenedOverlapping = KeyExtent.findOverlapping(extent, unopenedTablets);
+ Set<KeyExtent> openingOverlapping = KeyExtent.findOverlapping(extent, openingTablets);
+ Set<KeyExtent> onlineOverlapping = KeyExtent.findOverlapping(extent, onlineTablets);
+ Set<KeyExtent> all = new HashSet<KeyExtent>();
+ all.addAll(unopenedOverlapping);
+ all.addAll(openingOverlapping);
+ all.addAll(onlineOverlapping);
+
+ if (!all.isEmpty()) {
+ if (all.size() != 1 || !all.contains(extent)) {
+ log.error("Tablet " + extent + " overlaps previously assigned " + unopenedOverlapping + " " + openingOverlapping + " " + onlineOverlapping);
+ }
+ return;
+ }
+
+ unopenedTablets.add(extent);
+ }
+ }
+ }
+
+ // add the assignment job to the appropriate queue
+ log.info("Loading tablet " + extent);
+
+ final Runnable ah = new LoggingRunnable(log, new AssignmentHandler(extent));
+ // Root tablet assignment must take place immediately
+ if (extent.isRootTablet()) {
+ new Daemon("Root Tablet Assignment") {
+ @Override
+ public void run() {
+ ah.run();
+ if (onlineTablets.containsKey(extent)) {
+ log.info("Root tablet loaded: " + extent);
+ } else {
+ log.info("Root tablet failed to load");
+ }
+
+ }
+ }.start();
+ } else {
+ if (extent.isMeta()) {
+ resourceManager.addMetaDataAssignment(ah);
+ } else {
+ resourceManager.addAssignment(ah);
+ }
+ }
+ }
+
+ @Override
+ public void unloadTablet(TInfo tinfo, TCredentials credentials, String lock, TKeyExtent textent, boolean save) {
+ try {
+ checkPermission(credentials, lock, true, "unloadTablet");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+
+ KeyExtent extent = new KeyExtent(textent);
+
+ resourceManager.addMigration(extent, new LoggingRunnable(log, new UnloadTabletHandler(extent, save)));
+ }
+
+ @Override
+ public void flush(TInfo tinfo, TCredentials credentials, String lock, String tableId, ByteBuffer startRow, ByteBuffer endRow) {
+ try {
+ checkPermission(credentials, lock, true, "flush");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+
+ ArrayList<Tablet> tabletsToFlush = new ArrayList<Tablet>();
+
+ KeyExtent ke = new KeyExtent(new Text(tableId), ByteBufferUtil.toText(endRow), ByteBufferUtil.toText(startRow));
+
+ synchronized (onlineTablets) {
+ for (Tablet tablet : onlineTablets.values())
+ if (ke.overlaps(tablet.getExtent()))
+ tabletsToFlush.add(tablet);
+ }
+
+ Long flushID = null;
+
+ for (Tablet tablet : tabletsToFlush) {
+ if (flushID == null) {
+ // read the flush id once from zookeeper instead of reading
+ // it for each tablet
+ try {
+ flushID = tablet.getFlushID();
+ } catch (NoNodeException e) {
+ // table was probably deleted
+ log.info("Asked to flush table that has no flush id " + ke + " " + e.getMessage());
+ return;
+ }
+ }
+ tablet.flush(flushID);
+ }
+ }
+
+ @Override
+ public void flushTablet(TInfo tinfo, TCredentials credentials, String lock, TKeyExtent textent) throws TException {
+ try {
+ checkPermission(credentials, lock, true, "flushTablet");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+
+ Tablet tablet = onlineTablets.get(new KeyExtent(textent));
+ if (tablet != null) {
+ log.info("Flushing " + tablet.getExtent());
+ try {
+ tablet.flush(tablet.getFlushID());
+ } catch (NoNodeException nne) {
+ log.info("Asked to flush tablet that has no flush id " + new KeyExtent(textent) + " " + nne.getMessage());
+ }
+ }
+ }
+
+ @Override
+ public void halt(TInfo tinfo, TCredentials credentials, String lock) throws ThriftSecurityException {
+
+ checkPermission(credentials, lock, true, "halt");
+
+ Halt.halt(0, new Runnable() {
+ @Override
+ public void run() {
+ log.info("Master requested tablet server halt");
+ logGCInfo(getSystemConfiguration());
+ serverStopRequested = true;
+ try {
+ tabletServerLock.unlock();
+ } catch (Exception e) {
+ log.error(e, e);
+ }
+ }
+ });
+ }
+
+ @Override
+ public void fastHalt(TInfo info, TCredentials credentials, String lock) {
+ try {
+ halt(info, credentials, lock);
+ } catch (Exception e) {
+ log.warn("Error halting", e);
+ }
+ }
+
+ @Override
+ public TabletStats getHistoricalStats(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ return statsKeeper.getTabletStats();
+ }
+
+ @Override
+ public List<ActiveScan> getActiveScans(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ try {
+ checkPermission(credentials, null, true, "getScans");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw e;
+ }
+
+ return sessionManager.getActiveScans();
+ }
+
+ @Override
+ public void chop(TInfo tinfo, TCredentials credentials, String lock, TKeyExtent textent) throws TException {
+ try {
+ checkPermission(credentials, lock, true, "chop");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+
+ KeyExtent ke = new KeyExtent(textent);
+
+ Tablet tablet = onlineTablets.get(ke);
+ if (tablet != null) {
+ tablet.chopFiles();
+ }
+ }
+
+ @Override
+ public void compact(TInfo tinfo, TCredentials credentials, String lock, String tableId, ByteBuffer startRow, ByteBuffer endRow)
+ throws TException {
+ try {
+ checkPermission(credentials, lock, true, "compact");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+
+ KeyExtent ke = new KeyExtent(new Text(tableId), ByteBufferUtil.toText(endRow), ByteBufferUtil.toText(startRow));
+
+ ArrayList<Tablet> tabletsToCompact = new ArrayList<Tablet>();
+ synchronized (onlineTablets) {
+ for (Tablet tablet : onlineTablets.values())
+ if (ke.overlaps(tablet.getExtent()))
+ tabletsToCompact.add(tablet);
+ }
+
+ Long compactionId = null;
+
+ for (Tablet tablet : tabletsToCompact) {
+ // all for the same table id, so only need to read
+ // compaction id once
+ if (compactionId == null)
+ try {
+ compactionId = tablet.getCompactionID().getFirst();
+ } catch (NoNodeException e) {
+ log.info("Asked to compact table with no compaction id " + ke + " " + e.getMessage());
+ return;
+ }
+ tablet.compactAll(compactionId);
+ }
+
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface#removeLogs(org.apache.accumulo.trace.thrift.TInfo,
- * org.apache.accumulo.core.security.thrift.Credentials, java.util.List)
- */
+ @Override
+ public void removeLogs(TInfo tinfo, TCredentials credentials, List<String> filenames) throws TException {
+ String myname = getClientAddressString();
+ myname = myname.replace(':', '+');
+ Path logDir = new Path(Constants.getWalDirectory(acuConf), myname);
+ Set<String> loggers = new HashSet<String>();
+ logger.getLoggers(loggers);
+ nextFile: for (String filename : filenames) {
+ for (String logger : loggers) {
+ if (logger.contains(filename))
+ continue nextFile;
+ }
+ List<Tablet> onlineTabletsCopy = new ArrayList<Tablet>();
+ synchronized (onlineTablets) {
+ onlineTabletsCopy.addAll(onlineTablets.values());
+ }
+ for (Tablet tablet : onlineTabletsCopy) {
+ for (String current : tablet.getCurrentLogs()) {
+ if (current.contains(filename)) {
+ log.info("Attempted to delete " + filename + " from tablet " + tablet.getExtent());
+ continue nextFile;
+ }
+ }
+ }
+ try {
+ String source = logDir + "/" + filename;
+ if (acuConf.getBoolean(Property.TSERV_ARCHIVE_WALOGS)) {
+ String walogArchive = Constants.getBaseDir(acuConf) + "/walogArchive";
+ fs.mkdirs(new Path(walogArchive));
+ String dest = walogArchive + "/" + filename;
+ log.info("Archiving walog " + source + " to " + dest);
+ if (!fs.rename(new Path(source), new Path(dest)))
+ log.error("rename is unsuccessful");
+ } else {
+ log.info("Deleting walog " + filename);
+ Trash trash = new Trash(fs, fs.getConf());
+ Path sourcePath = new Path(source);
+ if (!(!acuConf.getBoolean(Property.GC_TRASH_IGNORE) && trash.moveToTrash(sourcePath)) && !fs.delete(sourcePath, true))
+ log.warn("Failed to delete walog " + source);
+ Path recoveryPath = new Path(Constants.getRecoveryDir(acuConf), filename);
+ try {
+ if (trash.moveToTrash(recoveryPath) || fs.delete(recoveryPath, true))
+ log.info("Deleted any recovery log " + filename);
+ } catch (FileNotFoundException ex) {
+ // ignore
+ }
+
+ }
+ } catch (IOException e) {
+ log.warn("Error attempting to delete write-ahead log " + filename + ": " + e);
+ }
+ }
+ }
+
+ @Override
+ public List<ActiveCompaction> getActiveCompactions(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ try {
+ checkPermission(credentials, null, true, "getActiveCompactions");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw e;
+ }
+
+ List<CompactionInfo> compactions = Compactor.getRunningCompactions();
+ List<ActiveCompaction> ret = new ArrayList<ActiveCompaction>(compactions.size());
+
+ for (CompactionInfo compactionInfo : compactions) {
+ ret.add(compactionInfo.toThrift());
+ }
+
+ return ret;
+ }
+ }
+
+ private class SplitRunner implements Runnable {
+ private Tablet tablet;
+
+ public SplitRunner(Tablet tablet) {
+ this.tablet = tablet;
+ }
+
+ @Override
+ public void run() {
+ if (majorCompactorDisabled) {
+ // this will make split task that were queued when shutdown was
+ // initiated exit
+ return;
+ }
+
+ splitTablet(tablet);
+ }
+ }
+
+ boolean isMajorCompactionDisabled() {
+ return majorCompactorDisabled;
+ }
+
+ void executeSplit(Tablet tablet) {
+ resourceManager.executeSplit(tablet.getExtent(), new LoggingRunnable(log, new SplitRunner(tablet)));
+ }
+
+ private class MajorCompactor implements Runnable {
+
+ @Override
+ public void run() {
+ while (!majorCompactorDisabled) {
+ try {
+ UtilWaitThread.sleep(getSystemConfiguration().getTimeInMillis(Property.TSERV_MAJC_DELAY));
+
+ TreeMap<KeyExtent,Tablet> copyOnlineTablets = new TreeMap<KeyExtent,Tablet>();
+
+ synchronized (onlineTablets) {
+ copyOnlineTablets.putAll(onlineTablets); // avoid
+ // concurrent
+ // modification
+ }
+
+ int numMajorCompactions
<TRUNCATED>
[38/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
index ad3d615,0000000..b9b68bb
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
@@@ -1,3620 -1,0 +1,3614 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.tabletserver;
+
+import static org.apache.accumulo.server.problems.ProblemType.TABLET_LOAD;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.lang.reflect.Field;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TimerTask;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.UUID;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.RunnableFuture;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.management.ObjectName;
+import javax.management.StandardMBean;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
+import org.apache.accumulo.core.client.impl.TabletType;
+import org.apache.accumulo.core.client.impl.Translator;
++import org.apache.accumulo.core.client.impl.Translators;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
- import org.apache.accumulo.core.client.impl.Translators;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.constraints.Constraint.Environment;
+import org.apache.accumulo.core.constraints.Violations;
+import org.apache.accumulo.core.data.Column;
+import org.apache.accumulo.core.data.ConstraintViolationSummary;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.thrift.InitialMultiScan;
+import org.apache.accumulo.core.data.thrift.InitialScan;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.data.thrift.MapFileInfo;
+import org.apache.accumulo.core.data.thrift.MultiScanResult;
+import org.apache.accumulo.core.data.thrift.ScanResult;
+import org.apache.accumulo.core.data.thrift.TColumn;
+import org.apache.accumulo.core.data.thrift.TKey;
+import org.apache.accumulo.core.data.thrift.TKeyExtent;
+import org.apache.accumulo.core.data.thrift.TKeyValue;
+import org.apache.accumulo.core.data.thrift.TMutation;
+import org.apache.accumulo.core.data.thrift.TRange;
+import org.apache.accumulo.core.data.thrift.UpdateErrors;
+import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.iterators.IterationInterruptedException;
+import org.apache.accumulo.core.master.thrift.Compacting;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.master.thrift.TableInfo;
+import org.apache.accumulo.core.master.thrift.TabletLoadState;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
+import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
+import org.apache.accumulo.core.tabletserver.thrift.NoSuchScanIDException;
+import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
+import org.apache.accumulo.core.tabletserver.thrift.ScanState;
+import org.apache.accumulo.core.tabletserver.thrift.ScanType;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Processor;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.LoggingRunnable;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.ServerServices;
+import org.apache.accumulo.core.util.ServerServices.Service;
+import org.apache.accumulo.core.util.SimpleThreadPool;
+import org.apache.accumulo.core.util.Stat;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockWatcher;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.client.ClientServiceHandler;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.conf.TableConfiguration;
+import org.apache.accumulo.server.data.ServerMutation;
+import org.apache.accumulo.server.logger.LogFileKey;
+import org.apache.accumulo.server.logger.LogFileValue;
+import org.apache.accumulo.server.master.state.Assignment;
+import org.apache.accumulo.server.master.state.DistributedStoreException;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletLocationState;
+import org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException;
+import org.apache.accumulo.server.master.state.TabletStateStore;
+import org.apache.accumulo.server.master.state.ZooTabletStateStore;
+import org.apache.accumulo.server.metrics.AbstractMetricsImpl;
+import org.apache.accumulo.server.problems.ProblemReport;
+import org.apache.accumulo.server.problems.ProblemReports;
+import org.apache.accumulo.server.security.AuditedSecurityOperation;
+import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.accumulo.server.security.SecurityOperation;
+import org.apache.accumulo.server.tabletserver.Compactor.CompactionInfo;
+import org.apache.accumulo.server.tabletserver.Tablet.CommitSession;
+import org.apache.accumulo.server.tabletserver.Tablet.KVEntry;
+import org.apache.accumulo.server.tabletserver.Tablet.LookupResult;
+import org.apache.accumulo.server.tabletserver.Tablet.MajorCompactionReason;
+import org.apache.accumulo.server.tabletserver.Tablet.MinorCompactionReason;
+import org.apache.accumulo.server.tabletserver.Tablet.ScanBatch;
+import org.apache.accumulo.server.tabletserver.Tablet.Scanner;
+import org.apache.accumulo.server.tabletserver.Tablet.SplitInfo;
+import org.apache.accumulo.server.tabletserver.Tablet.TConstraintViolationException;
+import org.apache.accumulo.server.tabletserver.Tablet.TabletClosedException;
+import org.apache.accumulo.server.tabletserver.TabletServerResourceManager.TabletResourceManager;
+import org.apache.accumulo.server.tabletserver.TabletStatsKeeper.Operation;
+import org.apache.accumulo.server.tabletserver.log.DfsLogger;
+import org.apache.accumulo.server.tabletserver.log.LogSorter;
+import org.apache.accumulo.server.tabletserver.log.MutationReceiver;
+import org.apache.accumulo.server.tabletserver.log.TabletServerLogger;
+import org.apache.accumulo.server.tabletserver.mastermessage.MasterMessage;
+import org.apache.accumulo.server.tabletserver.mastermessage.SplitReportMessage;
+import org.apache.accumulo.server.tabletserver.mastermessage.TabletStatusMessage;
+import org.apache.accumulo.server.tabletserver.metrics.TabletServerMBean;
+import org.apache.accumulo.server.tabletserver.metrics.TabletServerMinCMetrics;
+import org.apache.accumulo.server.tabletserver.metrics.TabletServerScanMetrics;
+import org.apache.accumulo.server.tabletserver.metrics.TabletServerUpdateMetrics;
+import org.apache.accumulo.server.trace.TraceFileSystem;
+import org.apache.accumulo.server.util.FileSystemMonitor;
+import org.apache.accumulo.server.util.Halt;
+import org.apache.accumulo.server.util.MapCounter;
+import org.apache.accumulo.server.util.MetadataTable;
+import org.apache.accumulo.server.util.MetadataTable.LogEntry;
+import org.apache.accumulo.server.util.TServerUtils;
+import org.apache.accumulo.server.util.TServerUtils.ServerPort;
+import org.apache.accumulo.server.util.time.RelativeTime;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
+import org.apache.accumulo.server.zookeeper.TransactionWatcher;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.start.Platform;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.accumulo.start.classloader.vfs.ContextManager;
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.instrument.Trace;
+import org.apache.accumulo.trace.instrument.thrift.TraceWrap;
+import org.apache.accumulo.trace.thrift.TInfo;
+import org.apache.commons.collections.map.LRUMap;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.Trash;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.SequenceFile.Reader;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+import org.apache.thrift.TProcessor;
+import org.apache.thrift.TServiceClient;
+import org.apache.thrift.server.TServer;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+
+enum ScanRunState {
+ QUEUED, RUNNING, FINISHED
+}
+
+public class TabletServer extends AbstractMetricsImpl implements org.apache.accumulo.server.tabletserver.metrics.TabletServerMBean {
+ private static final Logger log = Logger.getLogger(TabletServer.class);
+
+ private static HashMap<String,Long> prevGcTime = new HashMap<String,Long>();
+ private static long lastMemorySize = 0;
+ private static long gcTimeIncreasedCount;
+
+ private static final long MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS = 1000;
+
+ private TabletServerLogger logger;
+
+ protected TabletServerMinCMetrics mincMetrics = new TabletServerMinCMetrics();
+
+ private ServerConfiguration serverConfig;
+ private LogSorter logSorter = null;
+
+ public TabletServer(ServerConfiguration conf, FileSystem fs) {
+ super();
+ this.serverConfig = conf;
+ this.instance = conf.getInstance();
+ this.fs = TraceFileSystem.wrap(fs);
+ this.logSorter = new LogSorter(instance, fs, getSystemConfiguration());
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ synchronized (onlineTablets) {
+ long now = System.currentTimeMillis();
+ for (Tablet tablet : onlineTablets.values())
+ try {
+ tablet.updateRates(now);
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ }
+ }
+ }, 5000, 5000);
+ }
+
+ private synchronized static void logGCInfo(AccumuloConfiguration conf) {
+ List<GarbageCollectorMXBean> gcmBeans = ManagementFactory.getGarbageCollectorMXBeans();
+ Runtime rt = Runtime.getRuntime();
+
+ StringBuilder sb = new StringBuilder("gc");
+
+ boolean sawChange = false;
+
+ long maxIncreaseInCollectionTime = 0;
+
+ for (GarbageCollectorMXBean gcBean : gcmBeans) {
+ Long prevTime = prevGcTime.get(gcBean.getName());
+ long pt = 0;
+ if (prevTime != null) {
+ pt = prevTime;
+ }
+
+ long time = gcBean.getCollectionTime();
+
+ if (time - pt != 0) {
+ sawChange = true;
+ }
+
+ long increaseInCollectionTime = time - pt;
+ sb.append(String.format(" %s=%,.2f(+%,.2f) secs", gcBean.getName(), time / 1000.0, increaseInCollectionTime / 1000.0));
+ maxIncreaseInCollectionTime = Math.max(increaseInCollectionTime, maxIncreaseInCollectionTime);
+ prevGcTime.put(gcBean.getName(), time);
+ }
+
+ long mem = rt.freeMemory();
+ if (maxIncreaseInCollectionTime == 0) {
+ gcTimeIncreasedCount = 0;
+ } else {
+ gcTimeIncreasedCount++;
+ if (gcTimeIncreasedCount > 3 && mem < rt.maxMemory() * 0.05) {
+ log.warn("Running low on memory");
+ gcTimeIncreasedCount = 0;
+ }
+ }
+
+ if (mem > lastMemorySize) {
+ sawChange = true;
+ }
+
+ String sign = "+";
+ if (mem - lastMemorySize <= 0) {
+ sign = "";
+ }
+
+ sb.append(String.format(" freemem=%,d(%s%,d) totalmem=%,d", mem, sign, (mem - lastMemorySize), rt.totalMemory()));
+
+ if (sawChange) {
+ log.debug(sb.toString());
+ }
+
+ final long keepAliveTimeout = conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT);
+ if (maxIncreaseInCollectionTime > keepAliveTimeout) {
+ Halt.halt("Garbage collection may be interfering with lock keep-alive. Halting.", -1);
+ }
+
+ lastMemorySize = mem;
+ }
+
+ private TabletStatsKeeper statsKeeper;
+
+ private static class Session {
+ long lastAccessTime;
+ long startTime;
+ String user;
+ String client = TServerUtils.clientAddress.get();
+ public boolean reserved;
+
+ public void cleanup() {}
+ }
+
+ private static class SessionManager {
+
+ SecureRandom random;
+ Map<Long,Session> sessions;
+
+ SessionManager(AccumuloConfiguration conf) {
+ random = new SecureRandom();
+ sessions = new HashMap<Long,Session>();
+
+ final long maxIdle = conf.getTimeInMillis(Property.TSERV_SESSION_MAXIDLE);
+
+ Runnable r = new Runnable() {
+ @Override
+ public void run() {
+ sweep(maxIdle);
+ }
+ };
+
+ SimpleTimer.getInstance().schedule(r, 0, Math.max(maxIdle / 2, 1000));
+ }
+
+ synchronized long createSession(Session session, boolean reserve) {
+ long sid = random.nextLong();
+
+ while (sessions.containsKey(sid)) {
+ sid = random.nextLong();
+ }
+
+ sessions.put(sid, session);
+
+ session.reserved = reserve;
+
+ session.startTime = session.lastAccessTime = System.currentTimeMillis();
+
+ return sid;
+ }
+
+ /**
+ * while a session is reserved, it cannot be canceled or removed
+ *
+ * @param sessionId
+ */
+
+ synchronized Session reserveSession(long sessionId) {
+ Session session = sessions.get(sessionId);
+ if (session != null) {
+ if (session.reserved)
+ throw new IllegalStateException();
+ session.reserved = true;
+ }
+
+ return session;
+
+ }
+
+ synchronized void unreserveSession(Session session) {
+ if (!session.reserved)
+ throw new IllegalStateException();
+ session.reserved = false;
+ session.lastAccessTime = System.currentTimeMillis();
+ }
+
+ synchronized void unreserveSession(long sessionId) {
+ Session session = getSession(sessionId);
+ if (session != null)
+ unreserveSession(session);
+ }
+
+ synchronized Session getSession(long sessionId) {
+ Session session = sessions.get(sessionId);
+ if (session != null)
+ session.lastAccessTime = System.currentTimeMillis();
+ return session;
+ }
+
+ Session removeSession(long sessionId) {
+ Session session = null;
+ synchronized (this) {
+ session = sessions.remove(sessionId);
+ }
+
+ // do clean up out side of lock..
+ if (session != null)
+ session.cleanup();
+
+ return session;
+ }
+
+ private void sweep(long maxIdle) {
+ ArrayList<Session> sessionsToCleanup = new ArrayList<Session>();
+ synchronized (this) {
+ Iterator<Session> iter = sessions.values().iterator();
+ while (iter.hasNext()) {
+ Session session = iter.next();
+ long idleTime = System.currentTimeMillis() - session.lastAccessTime;
+ if (idleTime > maxIdle && !session.reserved) {
+ iter.remove();
+ sessionsToCleanup.add(session);
+ }
+ }
+ }
+
+ // do clean up outside of lock
+ for (Session session : sessionsToCleanup) {
+ session.cleanup();
+ }
+ }
+
+ synchronized void removeIfNotAccessed(final long sessionId, long delay) {
+ Session session = sessions.get(sessionId);
+ if (session != null) {
+ final long removeTime = session.lastAccessTime;
+ TimerTask r = new TimerTask() {
+ @Override
+ public void run() {
+ Session sessionToCleanup = null;
+ synchronized (SessionManager.this) {
+ Session session2 = sessions.get(sessionId);
+ if (session2 != null && session2.lastAccessTime == removeTime && !session2.reserved) {
+ sessions.remove(sessionId);
+ sessionToCleanup = session2;
+ }
+ }
+
+ // call clean up outside of lock
+ if (sessionToCleanup != null)
+ sessionToCleanup.cleanup();
+ }
+ };
+
+ SimpleTimer.getInstance().schedule(r, delay);
+ }
+ }
+
+ public synchronized Map<String,MapCounter<ScanRunState>> getActiveScansPerTable() {
+ Map<String,MapCounter<ScanRunState>> counts = new HashMap<String,MapCounter<ScanRunState>>();
+ for (Entry<Long,Session> entry : sessions.entrySet()) {
+
+ Session session = entry.getValue();
+ @SuppressWarnings("rawtypes")
+ ScanTask nbt = null;
+ String tableID = null;
+
+ if (session instanceof ScanSession) {
+ ScanSession ss = (ScanSession) session;
+ nbt = ss.nextBatchTask;
+ tableID = ss.extent.getTableId().toString();
+ } else if (session instanceof MultiScanSession) {
+ MultiScanSession mss = (MultiScanSession) session;
+ nbt = mss.lookupTask;
+ tableID = mss.threadPoolExtent.getTableId().toString();
+ }
+
+ if (nbt == null)
+ continue;
+
+ ScanRunState srs = nbt.getScanRunState();
+
+ if (srs == ScanRunState.FINISHED)
+ continue;
+
+ MapCounter<ScanRunState> stateCounts = counts.get(tableID);
+ if (stateCounts == null) {
+ stateCounts = new MapCounter<ScanRunState>();
+ counts.put(tableID, stateCounts);
+ }
+
+ stateCounts.increment(srs, 1);
+ }
+
+ return counts;
+ }
+
+ public synchronized List<ActiveScan> getActiveScans() {
+
+ ArrayList<ActiveScan> activeScans = new ArrayList<ActiveScan>();
+
+ long ct = System.currentTimeMillis();
+
+ for (Entry<Long,Session> entry : sessions.entrySet()) {
+ Session session = entry.getValue();
+ if (session instanceof ScanSession) {
+ ScanSession ss = (ScanSession) session;
+
+ ScanState state = ScanState.RUNNING;
+
+ ScanTask<ScanBatch> nbt = ss.nextBatchTask;
+ if (nbt == null) {
+ state = ScanState.IDLE;
+ } else {
+ switch (nbt.getScanRunState()) {
+ case QUEUED:
+ state = ScanState.QUEUED;
+ break;
+ case FINISHED:
+ state = ScanState.IDLE;
+ break;
+ case RUNNING:
+ default:
+ /* do nothing */
+ break;
+ }
+ }
+
+ activeScans.add(new ActiveScan(ss.client, ss.user, ss.extent.getTableId().toString(), ct - ss.startTime, ct - ss.lastAccessTime, ScanType.SINGLE,
+ state, ss.extent.toThrift(), Translator.translate(ss.columnSet, Translators.CT), ss.ssiList, ss.ssio, ss.auths.getAuthorizationsBB()));
+
+ } else if (session instanceof MultiScanSession) {
+ MultiScanSession mss = (MultiScanSession) session;
+
+ ScanState state = ScanState.RUNNING;
+
+ ScanTask<MultiScanResult> nbt = mss.lookupTask;
+ if (nbt == null) {
+ state = ScanState.IDLE;
+ } else {
+ switch (nbt.getScanRunState()) {
+ case QUEUED:
+ state = ScanState.QUEUED;
+ break;
+ case FINISHED:
+ state = ScanState.IDLE;
+ break;
+ case RUNNING:
+ default:
+ /* do nothing */
+ break;
+ }
+ }
+
+ activeScans.add(new ActiveScan(mss.client, mss.user, mss.threadPoolExtent.getTableId().toString(), ct - mss.startTime, ct - mss.lastAccessTime,
+ ScanType.BATCH, state, mss.threadPoolExtent.toThrift(), Translator.translate(mss.columnSet, Translators.CT), mss.ssiList, mss.ssio, mss.auths
+ .getAuthorizationsBB()));
+ }
+ }
+
+ return activeScans;
+ }
+ }
+
+ static class TservConstraintEnv implements Environment {
+
+ private TCredentials credentials;
+ private SecurityOperation security;
+ private Authorizations auths;
+ private KeyExtent ke;
+
+ TservConstraintEnv(SecurityOperation secOp, TCredentials credentials) {
+ this.security = secOp;
+ this.credentials = credentials;
+ }
+
+ void setExtent(KeyExtent ke) {
+ this.ke = ke;
+ }
+
+ @Override
+ public KeyExtent getExtent() {
+ return ke;
+ }
+
+ @Override
+ public String getUser() {
+ return credentials.getPrincipal();
+ }
+
+ @Override
+ public Authorizations getAuthorizations() {
+ if (auths == null)
+ try {
+ this.auths = security.getUserAuthorizations(credentials);
+ } catch (ThriftSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ return auths;
+ }
+
+ }
+
+ private abstract class ScanTask<T> implements RunnableFuture<T> {
+
+ protected AtomicBoolean interruptFlag;
+ protected ArrayBlockingQueue<Object> resultQueue;
+ protected AtomicInteger state;
+ protected AtomicReference<ScanRunState> runState;
+
+ private static final int INITIAL = 1;
+ private static final int ADDED = 2;
+ private static final int CANCELED = 3;
+
+ ScanTask() {
+ interruptFlag = new AtomicBoolean(false);
+ runState = new AtomicReference<ScanRunState>(ScanRunState.QUEUED);
+ state = new AtomicInteger(INITIAL);
+ resultQueue = new ArrayBlockingQueue<Object>(1);
+ }
+
+ protected void addResult(Object o) {
+ if (state.compareAndSet(INITIAL, ADDED))
+ resultQueue.add(o);
+ else if (state.get() == ADDED)
+ throw new IllegalStateException("Tried to add more than one result");
+ }
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ if (!mayInterruptIfRunning)
+ throw new IllegalArgumentException("Cancel will always attempt to interupt running next batch task");
+
+ if (state.get() == CANCELED)
+ return true;
+
+ if (state.compareAndSet(INITIAL, CANCELED)) {
+ interruptFlag.set(true);
+ resultQueue = null;
+ return true;
+ }
+
+ return false;
+ }
+
+ @Override
+ public T get() throws InterruptedException, ExecutionException {
+ throw new UnsupportedOperationException();
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
+
+ ArrayBlockingQueue<Object> localRQ = resultQueue;
+
+ if (state.get() == CANCELED)
+ throw new CancellationException();
+
+ if (localRQ == null && state.get() == ADDED)
+ throw new IllegalStateException("Tried to get result twice");
+
+ Object r = localRQ.poll(timeout, unit);
+
+ // could have been canceled while waiting
+ if (state.get() == CANCELED) {
+ if (r != null)
+ throw new IllegalStateException("Nothing should have been added when in canceled state");
+
+ throw new CancellationException();
+ }
+
+ if (r == null)
+ throw new TimeoutException();
+
+ // make this method stop working now that something is being
+ // returned
+ resultQueue = null;
+
+ if (r instanceof Throwable)
+ throw new ExecutionException((Throwable) r);
+
+ return (T) r;
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return state.get() == CANCELED;
+ }
+
+ @Override
+ public boolean isDone() {
+ return runState.get().equals(ScanRunState.FINISHED);
+ }
+
+ public ScanRunState getScanRunState() {
+ return runState.get();
+ }
+
+ }
+
+ private static class UpdateSession extends Session {
+ public Tablet currentTablet;
+ public MapCounter<Tablet> successfulCommits = new MapCounter<Tablet>();
+ Map<KeyExtent,Long> failures = new HashMap<KeyExtent,Long>();
+ HashMap<KeyExtent,SecurityErrorCode> authFailures = new HashMap<KeyExtent,SecurityErrorCode>();
+ public Violations violations;
+ public TCredentials credentials;
+ public long totalUpdates = 0;
+ public long flushTime = 0;
+ Stat prepareTimes = new Stat();
+ Stat walogTimes = new Stat();
+ Stat commitTimes = new Stat();
+ Stat authTimes = new Stat();
+ public Map<Tablet,List<Mutation>> queuedMutations = new HashMap<Tablet,List<Mutation>>();
+ public long queuedMutationSize = 0;
+ TservConstraintEnv cenv = null;
+ }
+
+ private static class ScanSession extends Session {
+ public KeyExtent extent;
+ public HashSet<Column> columnSet;
+ public List<IterInfo> ssiList;
+ public Map<String,Map<String,String>> ssio;
+ public Authorizations auths;
+ public long entriesReturned = 0;
+ public Stat nbTimes = new Stat();
+ public long batchCount = 0;
+ public volatile ScanTask<ScanBatch> nextBatchTask;
+ public AtomicBoolean interruptFlag;
+ public Scanner scanner;
+
+ @Override
+ public void cleanup() {
+ try {
+ if (nextBatchTask != null)
+ nextBatchTask.cancel(true);
+ } finally {
+ if (scanner != null)
+ scanner.close();
+ }
+ }
+
+ }
+
+ private static class MultiScanSession extends Session {
+ HashSet<Column> columnSet;
+ Map<KeyExtent,List<Range>> queries;
+ public List<IterInfo> ssiList;
+ public Map<String,Map<String,String>> ssio;
+ public Authorizations auths;
+
+ // stats
+ int numRanges;
+ int numTablets;
+ int numEntries;
+ long totalLookupTime;
+
+ public volatile ScanTask<MultiScanResult> lookupTask;
+ public KeyExtent threadPoolExtent;
+
+ @Override
+ public void cleanup() {
+ if (lookupTask != null)
+ lookupTask.cancel(true);
+ }
+ }
+
+ /**
+ * This little class keeps track of writes in progress and allows readers to wait for writes that started before the read. It assumes that the operation ids
+ * are monotonically increasing.
+ *
+ */
+ static class WriteTracker {
+ private static AtomicLong operationCounter = new AtomicLong(1);
+ private Map<TabletType,TreeSet<Long>> inProgressWrites = new EnumMap<TabletType,TreeSet<Long>>(TabletType.class);
+
+ WriteTracker() {
+ for (TabletType ttype : TabletType.values()) {
+ inProgressWrites.put(ttype, new TreeSet<Long>());
+ }
+ }
+
+ synchronized long startWrite(TabletType ttype) {
+ long operationId = operationCounter.getAndIncrement();
+ inProgressWrites.get(ttype).add(operationId);
+ return operationId;
+ }
+
+ synchronized void finishWrite(long operationId) {
+ if (operationId == -1)
+ return;
+
+ boolean removed = false;
+
+ for (TabletType ttype : TabletType.values()) {
+ removed = inProgressWrites.get(ttype).remove(operationId);
+ if (removed)
+ break;
+ }
+
+ if (!removed) {
+ throw new IllegalArgumentException("Attempted to finish write not in progress, operationId " + operationId);
+ }
+
+ this.notifyAll();
+ }
+
+ synchronized void waitForWrites(TabletType ttype) {
+ long operationId = operationCounter.getAndIncrement();
+ while (inProgressWrites.get(ttype).floor(operationId) != null) {
+ try {
+ this.wait();
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ }
+ }
+ }
+
+ public long startWrite(Set<Tablet> keySet) {
+ if (keySet.size() == 0)
+ return -1;
+
+ ArrayList<KeyExtent> extents = new ArrayList<KeyExtent>(keySet.size());
+
+ for (Tablet tablet : keySet)
+ extents.add(tablet.getExtent());
+
+ return startWrite(TabletType.type(extents));
+ }
+ }
+
+ public AccumuloConfiguration getSystemConfiguration() {
+ return serverConfig.getConfiguration();
+ }
+
+ TransactionWatcher watcher = new TransactionWatcher();
+
+ private class ThriftClientHandler extends ClientServiceHandler implements TabletClientService.Iface {
+
+ SessionManager sessionManager;
+
+ AccumuloConfiguration acuConf = getSystemConfiguration();
+
+ TabletServerUpdateMetrics updateMetrics = new TabletServerUpdateMetrics();
+
+ TabletServerScanMetrics scanMetrics = new TabletServerScanMetrics();
+
+ WriteTracker writeTracker = new WriteTracker();
+
+ ThriftClientHandler() {
+ super(instance, watcher);
+ log.debug(ThriftClientHandler.class.getName() + " created");
+ sessionManager = new SessionManager(getSystemConfiguration());
+ // Register the metrics MBean
+ try {
+ updateMetrics.register();
+ scanMetrics.register();
+ } catch (Exception e) {
+ log.error("Exception registering MBean with MBean Server", e);
+ }
+ }
+
+ @Override
+ public List<TKeyExtent> bulkImport(TInfo tinfo, TCredentials credentials, long tid, Map<TKeyExtent,Map<String,MapFileInfo>> files, boolean setTime)
+ throws ThriftSecurityException {
+
+ if (!security.canPerformSystemActions(credentials))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ List<TKeyExtent> failures = new ArrayList<TKeyExtent>();
+
+ for (Entry<TKeyExtent,Map<String,MapFileInfo>> entry : files.entrySet()) {
+ TKeyExtent tke = entry.getKey();
+ Map<String,MapFileInfo> fileMap = entry.getValue();
+
+ Tablet importTablet = onlineTablets.get(new KeyExtent(tke));
+
+ if (importTablet == null) {
+ failures.add(tke);
+ } else {
+ try {
+ importTablet.importMapFiles(tid, fileMap, setTime);
+ } catch (IOException ioe) {
+ log.info("files " + fileMap.keySet() + " not imported to " + new KeyExtent(tke) + ": " + ioe.getMessage());
+ failures.add(tke);
+ }
+ }
+ }
+ return failures;
+ }
+
+ private class NextBatchTask extends ScanTask<ScanBatch> {
+
+ private long scanID;
+
+ NextBatchTask(long scanID, AtomicBoolean interruptFlag) {
+ this.scanID = scanID;
+ this.interruptFlag = interruptFlag;
+
+ if (interruptFlag.get())
+ cancel(true);
+ }
+
+ @Override
+ public void run() {
+
+ final ScanSession scanSession = (ScanSession) sessionManager.getSession(scanID);
+ String oldThreadName = Thread.currentThread().getName();
+
+ try {
+ if (isCancelled() || scanSession == null)
+ return;
+
+ runState.set(ScanRunState.RUNNING);
+
+ Thread.currentThread().setName(
+ "User: " + scanSession.user + " Start: " + scanSession.startTime + " Client: " + scanSession.client + " Tablet: " + scanSession.extent);
+
+ Tablet tablet = onlineTablets.get(scanSession.extent);
+
+ if (tablet == null) {
+ addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
+ return;
+ }
+
+ long t1 = System.currentTimeMillis();
+ ScanBatch batch = scanSession.scanner.read();
+ long t2 = System.currentTimeMillis();
+ scanSession.nbTimes.addStat(t2 - t1);
+
+ // there should only be one thing on the queue at a time, so
+ // it should be ok to call add()
+ // instead of put()... if add() fails because queue is at
+ // capacity it means there is code
+ // problem somewhere
+ addResult(batch);
+ } catch (TabletClosedException e) {
+ addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
+ } catch (IterationInterruptedException iie) {
+ if (!isCancelled()) {
+ log.warn("Iteration interrupted, when scan not cancelled", iie);
+ addResult(iie);
+ }
+ } catch (TooManyFilesException tmfe) {
+ addResult(tmfe);
+ } catch (Throwable e) {
+ log.warn("exception while scanning tablet " + (scanSession == null ? "(unknown)" : scanSession.extent), e);
+ addResult(e);
+ } finally {
+ runState.set(ScanRunState.FINISHED);
+ Thread.currentThread().setName(oldThreadName);
+ }
+
+ }
+ }
+
+ private class LookupTask extends ScanTask<MultiScanResult> {
+
+ private long scanID;
+
+ LookupTask(long scanID) {
+ this.scanID = scanID;
+ }
+
+ @Override
+ public void run() {
+ MultiScanSession session = (MultiScanSession) sessionManager.getSession(scanID);
+ String oldThreadName = Thread.currentThread().getName();
+
+ try {
+ if (isCancelled() || session == null)
+ return;
+
+ TableConfiguration acuTableConf = ServerConfiguration.getTableConfiguration(instance, session.threadPoolExtent.getTableId().toString());
+ long maxResultsSize = acuTableConf.getMemoryInBytes(Property.TABLE_SCAN_MAXMEM);
+
+ runState.set(ScanRunState.RUNNING);
+ Thread.currentThread().setName("Client: " + session.client + " User: " + session.user + " Start: " + session.startTime + " Table: ");
+
+ long bytesAdded = 0;
+ long maxScanTime = 4000;
+
+ long startTime = System.currentTimeMillis();
+
+ ArrayList<KVEntry> results = new ArrayList<KVEntry>();
+ Map<KeyExtent,List<Range>> failures = new HashMap<KeyExtent,List<Range>>();
+ ArrayList<KeyExtent> fullScans = new ArrayList<KeyExtent>();
+ KeyExtent partScan = null;
+ Key partNextKey = null;
+ boolean partNextKeyInclusive = false;
+
+ Iterator<Entry<KeyExtent,List<Range>>> iter = session.queries.entrySet().iterator();
+
+ // check the time so that the read ahead thread is not monopolized
+ while (iter.hasNext() && bytesAdded < maxResultsSize && (System.currentTimeMillis() - startTime) < maxScanTime) {
+ Entry<KeyExtent,List<Range>> entry = iter.next();
+
+ iter.remove();
+
+ // check that tablet server is serving requested tablet
+ Tablet tablet = onlineTablets.get(entry.getKey());
+ if (tablet == null) {
+ failures.put(entry.getKey(), entry.getValue());
+ continue;
+ }
+ Thread.currentThread().setName(
+ "Client: " + session.client + " User: " + session.user + " Start: " + session.startTime + " Tablet: " + entry.getKey().toString());
+
+ LookupResult lookupResult;
+ try {
+
+ // do the following check to avoid a race condition
+ // between setting false below and the task being
+ // canceled
+ if (isCancelled())
+ interruptFlag.set(true);
+
+ lookupResult = tablet.lookup(entry.getValue(), session.columnSet, session.auths, results, maxResultsSize - bytesAdded, session.ssiList,
+ session.ssio, interruptFlag);
+
+ // if the tablet was closed it it possible that the
+ // interrupt flag was set.... do not want it set for
+ // the next
+ // lookup
+ interruptFlag.set(false);
+
+ } catch (IOException e) {
+ log.warn("lookup failed for tablet " + entry.getKey(), e);
+ throw new RuntimeException(e);
+ }
+
+ bytesAdded += lookupResult.bytesAdded;
+
+ if (lookupResult.unfinishedRanges.size() > 0) {
+ if (lookupResult.closed) {
+ failures.put(entry.getKey(), lookupResult.unfinishedRanges);
+ } else {
+ session.queries.put(entry.getKey(), lookupResult.unfinishedRanges);
+ partScan = entry.getKey();
+ partNextKey = lookupResult.unfinishedRanges.get(0).getStartKey();
+ partNextKeyInclusive = lookupResult.unfinishedRanges.get(0).isStartKeyInclusive();
+ }
+ } else {
+ fullScans.add(entry.getKey());
+ }
+ }
+
+ long finishTime = System.currentTimeMillis();
+ session.totalLookupTime += (finishTime - startTime);
+ session.numEntries += results.size();
+
+ // convert everything to thrift before adding result
+ List<TKeyValue> retResults = new ArrayList<TKeyValue>();
+ for (KVEntry entry : results)
+ retResults.add(new TKeyValue(entry.key.toThrift(), ByteBuffer.wrap(entry.value)));
+ Map<TKeyExtent,List<TRange>> retFailures = Translator.translate(failures, Translators.KET, new Translator.ListTranslator<Range,TRange>(Translators.RT));
+ List<TKeyExtent> retFullScans = Translator.translate(fullScans, Translators.KET);
+ TKeyExtent retPartScan = null;
+ TKey retPartNextKey = null;
+ if (partScan != null) {
+ retPartScan = partScan.toThrift();
+ retPartNextKey = partNextKey.toThrift();
+ }
+ // add results to queue
+ addResult(new MultiScanResult(retResults, retFailures, retFullScans, retPartScan, retPartNextKey, partNextKeyInclusive, session.queries.size() != 0));
+ } catch (IterationInterruptedException iie) {
+ if (!isCancelled()) {
+ log.warn("Iteration interrupted, when scan not cancelled", iie);
+ addResult(iie);
+ }
+ } catch (Throwable e) {
+ log.warn("exception while doing multi-scan ", e);
+ addResult(e);
+ } finally {
+ Thread.currentThread().setName(oldThreadName);
+ runState.set(ScanRunState.FINISHED);
+ }
+ }
+ }
+
+ @Override
+ public InitialScan startScan(TInfo tinfo, TCredentials credentials, TKeyExtent textent, TRange range, List<TColumn> columns, int batchSize,
+ List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites, boolean isolated)
+ throws NotServingTabletException, ThriftSecurityException, org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
+
+ Authorizations userauths = null;
+ if (!security.canScan(credentials, new String(textent.getTable(), Constants.UTF8)))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ userauths = security.getUserAuthorizations(credentials);
+ for (ByteBuffer auth : authorizations)
+ if (!userauths.contains(ByteBufferUtil.toBytes(auth)))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
+
+ KeyExtent extent = new KeyExtent(textent);
+
+ // wait for any writes that are in flight.. this done to ensure
+ // consistency across client restarts... assume a client writes
+ // to accumulo and dies while waiting for a confirmation from
+ // accumulo... the client process restarts and tries to read
+ // data from accumulo making the assumption that it will get
+ // any writes previously made... however if the server side thread
+ // processing the write from the dead client is still in progress,
+ // the restarted client may not see the write unless we wait here.
+ // this behavior is very important when the client is reading the
+ // !METADATA table
+ if (waitForWrites)
+ writeTracker.waitForWrites(TabletType.type(extent));
+
+ Tablet tablet = onlineTablets.get(extent);
+ if (tablet == null)
+ throw new NotServingTabletException(textent);
+
+ ScanSession scanSession = new ScanSession();
+ scanSession.user = credentials.getPrincipal();
+ scanSession.extent = new KeyExtent(extent);
+ scanSession.columnSet = new HashSet<Column>();
+ scanSession.ssiList = ssiList;
+ scanSession.ssio = ssio;
+ scanSession.auths = new Authorizations(authorizations);
+ scanSession.interruptFlag = new AtomicBoolean();
+
+ for (TColumn tcolumn : columns) {
+ scanSession.columnSet.add(new Column(tcolumn));
+ }
+
+ scanSession.scanner = tablet.createScanner(new Range(range), batchSize, scanSession.columnSet, scanSession.auths, ssiList, ssio, isolated,
+ scanSession.interruptFlag);
+
+ long sid = sessionManager.createSession(scanSession, true);
+
+ ScanResult scanResult;
+ try {
+ scanResult = continueScan(tinfo, sid, scanSession);
+ } catch (NoSuchScanIDException e) {
+ log.error("The impossible happened", e);
+ throw new RuntimeException();
+ } finally {
+ sessionManager.unreserveSession(sid);
+ }
+
+ return new InitialScan(sid, scanResult);
+ }
+
+ @Override
+ public ScanResult continueScan(TInfo tinfo, long scanID) throws NoSuchScanIDException, NotServingTabletException,
+ org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
+ ScanSession scanSession = (ScanSession) sessionManager.reserveSession(scanID);
+ if (scanSession == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ try {
+ return continueScan(tinfo, scanID, scanSession);
+ } finally {
+ sessionManager.unreserveSession(scanSession);
+ }
+ }
+
+ private ScanResult continueScan(TInfo tinfo, long scanID, ScanSession scanSession) throws NoSuchScanIDException, NotServingTabletException,
+ org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
+
+ if (scanSession.nextBatchTask == null) {
+ scanSession.nextBatchTask = new NextBatchTask(scanID, scanSession.interruptFlag);
+ resourceManager.executeReadAhead(scanSession.extent, scanSession.nextBatchTask);
+ }
+
+ ScanBatch bresult;
+ try {
+ bresult = scanSession.nextBatchTask.get(MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS, TimeUnit.MILLISECONDS);
+ scanSession.nextBatchTask = null;
+ } catch (ExecutionException e) {
+ sessionManager.removeSession(scanID);
+ if (e.getCause() instanceof NotServingTabletException)
+ throw (NotServingTabletException) e.getCause();
+ else if (e.getCause() instanceof TooManyFilesException)
+ throw new org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException(scanSession.extent.toThrift());
+ else
+ throw new RuntimeException(e);
+ } catch (CancellationException ce) {
+ sessionManager.removeSession(scanID);
+ Tablet tablet = onlineTablets.get(scanSession.extent);
+ if (tablet == null || tablet.isClosed())
+ throw new NotServingTabletException(scanSession.extent.toThrift());
+ else
+ throw new NoSuchScanIDException();
+ } catch (TimeoutException e) {
+ List<TKeyValue> param = Collections.emptyList();
+ long timeout = acuConf.getTimeInMillis(Property.TSERV_CLIENT_TIMEOUT);
+ sessionManager.removeIfNotAccessed(scanID, timeout);
+ return new ScanResult(param, true);
+ } catch (Throwable t) {
+ sessionManager.removeSession(scanID);
+ log.warn("Failed to get next batch", t);
+ throw new RuntimeException(t);
+ }
+
+ ScanResult scanResult = new ScanResult(Key.compress(bresult.results), bresult.more);
+
+ scanSession.entriesReturned += scanResult.results.size();
+
+ scanSession.batchCount++;
+
+ if (scanResult.more && scanSession.batchCount > 3) {
+ // start reading next batch while current batch is transmitted
+ // to client
+ scanSession.nextBatchTask = new NextBatchTask(scanID, scanSession.interruptFlag);
+ resourceManager.executeReadAhead(scanSession.extent, scanSession.nextBatchTask);
+ }
+
+ if (!scanResult.more)
+ closeScan(tinfo, scanID);
+
+ return scanResult;
+ }
+
+ @Override
+ public void closeScan(TInfo tinfo, long scanID) {
+ ScanSession ss = (ScanSession) sessionManager.removeSession(scanID);
+ if (ss != null) {
+ long t2 = System.currentTimeMillis();
+
+ log.debug(String.format("ScanSess tid %s %s %,d entries in %.2f secs, nbTimes = [%s] ", TServerUtils.clientAddress.get(), ss.extent.getTableId()
+ .toString(), ss.entriesReturned, (t2 - ss.startTime) / 1000.0, ss.nbTimes.toString()));
+ if (scanMetrics.isEnabled()) {
+ scanMetrics.add(TabletServerScanMetrics.scan, t2 - ss.startTime);
+ scanMetrics.add(TabletServerScanMetrics.resultSize, ss.entriesReturned);
+ }
+ }
+ }
+
+ @Override
+ public InitialMultiScan startMultiScan(TInfo tinfo, TCredentials credentials, Map<TKeyExtent,List<TRange>> tbatch, List<TColumn> tcolumns,
+ List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites) throws ThriftSecurityException {
+ // find all of the tables that need to be scanned
+ HashSet<String> tables = new HashSet<String>();
+ for (TKeyExtent keyExtent : tbatch.keySet()) {
+ tables.add(new String(keyExtent.getTable(), Constants.UTF8));
+ }
+
+ // check if user has permission to the tables
+ Authorizations userauths = null;
+ for (String table : tables)
+ if (!security.canScan(credentials, table))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ userauths = security.getUserAuthorizations(credentials);
+ for (ByteBuffer auth : authorizations)
+ if (!userauths.contains(ByteBufferUtil.toBytes(auth)))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
+
+ KeyExtent threadPoolExtent = null;
+
+ Map<KeyExtent,List<Range>> batch = Translator.translate(tbatch, Translators.TKET, new Translator.ListTranslator<TRange,Range>(Translators.TRT));
+
+ for (KeyExtent keyExtent : batch.keySet()) {
+ if (threadPoolExtent == null) {
+ threadPoolExtent = keyExtent;
+ } else if (keyExtent.isRootTablet()) {
+ throw new IllegalArgumentException("Cannot batch query root tablet with other tablets " + threadPoolExtent + " " + keyExtent);
+ } else if (keyExtent.isMeta() && !threadPoolExtent.isMeta()) {
+ throw new IllegalArgumentException("Cannot batch query !METADATA and non !METADATA tablets " + threadPoolExtent + " " + keyExtent);
+ }
+
+ }
+
+ if (waitForWrites)
+ writeTracker.waitForWrites(TabletType.type(batch.keySet()));
+
+ MultiScanSession mss = new MultiScanSession();
+ mss.user = credentials.getPrincipal();
+ mss.queries = batch;
+ mss.columnSet = new HashSet<Column>(tcolumns.size());
+ mss.ssiList = ssiList;
+ mss.ssio = ssio;
+ mss.auths = new Authorizations(authorizations);
+
+ mss.numTablets = batch.size();
+ for (List<Range> ranges : batch.values()) {
+ mss.numRanges += ranges.size();
+ }
+
+ for (TColumn tcolumn : tcolumns)
+ mss.columnSet.add(new Column(tcolumn));
+
+ mss.threadPoolExtent = threadPoolExtent;
+
+ long sid = sessionManager.createSession(mss, true);
+
+ MultiScanResult result;
+ try {
+ result = continueMultiScan(tinfo, sid, mss);
+ } catch (NoSuchScanIDException e) {
+ log.error("the impossible happened", e);
+ throw new RuntimeException("the impossible happened", e);
+ } finally {
+ sessionManager.unreserveSession(sid);
+ }
+
+ return new InitialMultiScan(sid, result);
+ }
+
+ @Override
+ public MultiScanResult continueMultiScan(TInfo tinfo, long scanID) throws NoSuchScanIDException {
+
+ MultiScanSession session = (MultiScanSession) sessionManager.reserveSession(scanID);
+
+ if (session == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ try {
+ return continueMultiScan(tinfo, scanID, session);
+ } finally {
+ sessionManager.unreserveSession(session);
+ }
+ }
+
+ private MultiScanResult continueMultiScan(TInfo tinfo, long scanID, MultiScanSession session) throws NoSuchScanIDException {
+
+ if (session.lookupTask == null) {
+ session.lookupTask = new LookupTask(scanID);
+ resourceManager.executeReadAhead(session.threadPoolExtent, session.lookupTask);
+ }
+
+ try {
+ MultiScanResult scanResult = session.lookupTask.get(MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS, TimeUnit.MILLISECONDS);
+ session.lookupTask = null;
+ return scanResult;
+ } catch (TimeoutException e1) {
+ long timeout = acuConf.getTimeInMillis(Property.TSERV_CLIENT_TIMEOUT);
+ sessionManager.removeIfNotAccessed(scanID, timeout);
+ List<TKeyValue> results = Collections.emptyList();
+ Map<TKeyExtent,List<TRange>> failures = Collections.emptyMap();
+ List<TKeyExtent> fullScans = Collections.emptyList();
+ return new MultiScanResult(results, failures, fullScans, null, null, false, true);
+ } catch (Throwable t) {
+ sessionManager.removeSession(scanID);
+ log.warn("Failed to get multiscan result", t);
+ throw new RuntimeException(t);
+ }
+ }
+
+ @Override
+ public void closeMultiScan(TInfo tinfo, long scanID) throws NoSuchScanIDException {
+ MultiScanSession session = (MultiScanSession) sessionManager.removeSession(scanID);
+ if (session == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ long t2 = System.currentTimeMillis();
+ log.debug(String.format("MultiScanSess %s %,d entries in %.2f secs (lookup_time:%.2f secs tablets:%,d ranges:%,d) ", TServerUtils.clientAddress.get(),
+ session.numEntries, (t2 - session.startTime) / 1000.0, session.totalLookupTime / 1000.0, session.numTablets, session.numRanges));
+ }
+
+ @Override
+ public long startUpdate(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException {
+ // Make sure user is real
+
+ security.authenticateUser(credentials, credentials);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
+
+ UpdateSession us = new UpdateSession();
+ us.violations = new Violations();
+ us.credentials = credentials;
+ us.cenv = new TservConstraintEnv(security, us.credentials);
+
+ long sid = sessionManager.createSession(us, false);
+
+ return sid;
+ }
+
+ private void setUpdateTablet(UpdateSession us, KeyExtent keyExtent) {
+ long t1 = System.currentTimeMillis();
+ if (us.currentTablet != null && us.currentTablet.getExtent().equals(keyExtent))
+ return;
+ if (us.currentTablet == null && (us.failures.containsKey(keyExtent) || us.authFailures.containsKey(keyExtent))) {
+ // if there were previous failures, then do not accept additional writes
+ return;
+ }
+
+ try {
+ // if user has no permission to write to this table, add it to
+ // the failures list
+ boolean sameTable = us.currentTablet != null && (us.currentTablet.getExtent().getTableId().equals(keyExtent.getTableId()));
+ if (sameTable || security.canWrite(us.credentials, keyExtent.getTableId().toString())) {
+ long t2 = System.currentTimeMillis();
+ us.authTimes.addStat(t2 - t1);
+ us.currentTablet = onlineTablets.get(keyExtent);
+ if (us.currentTablet != null) {
+ us.queuedMutations.put(us.currentTablet, new ArrayList<Mutation>());
+ } else {
+ // not serving tablet, so report all mutations as
+ // failures
+ us.failures.put(keyExtent, 0l);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.unknownTabletErrors, 0);
+ }
+ } else {
+ log.warn("Denying access to table " + keyExtent.getTableId() + " for user " + us.credentials.getPrincipal());
+ long t2 = System.currentTimeMillis();
+ us.authTimes.addStat(t2 - t1);
+ us.currentTablet = null;
+ us.authFailures.put(keyExtent, SecurityErrorCode.PERMISSION_DENIED);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
+ return;
+ }
+ } catch (ThriftSecurityException e) {
+ log.error("Denying permission to check user " + us.credentials.getPrincipal() + " with user " + e.getUser(), e);
+ long t2 = System.currentTimeMillis();
+ us.authTimes.addStat(t2 - t1);
+ us.currentTablet = null;
+ us.authFailures.put(keyExtent, e.getCode());
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
+ return;
+ }
+ }
+
+ @Override
+ public void applyUpdates(TInfo tinfo, long updateID, TKeyExtent tkeyExtent, List<TMutation> tmutations) {
+ UpdateSession us = (UpdateSession) sessionManager.reserveSession(updateID);
+ if (us == null) {
+ throw new RuntimeException("No Such SessionID");
+ }
+
+ try {
+ KeyExtent keyExtent = new KeyExtent(tkeyExtent);
+ setUpdateTablet(us, keyExtent);
+
+ if (us.currentTablet != null) {
+ List<Mutation> mutations = us.queuedMutations.get(us.currentTablet);
+ for (TMutation tmutation : tmutations) {
+ Mutation mutation = new ServerMutation(tmutation);
+ mutations.add(mutation);
+ us.queuedMutationSize += mutation.numBytes();
+ }
+ if (us.queuedMutationSize > getSystemConfiguration().getMemoryInBytes(Property.TSERV_MUTATION_QUEUE_MAX))
+ flush(us);
+ }
+ } finally {
+ sessionManager.unreserveSession(us);
+ }
+ }
+
+ private void flush(UpdateSession us) {
+
+ int mutationCount = 0;
+ Map<CommitSession,List<Mutation>> sendables = new HashMap<CommitSession,List<Mutation>>();
+ Throwable error = null;
+
+ long pt1 = System.currentTimeMillis();
+
+ boolean containsMetadataTablet = false;
+ for (Tablet tablet : us.queuedMutations.keySet())
+ if (tablet.getExtent().isMeta())
+ containsMetadataTablet = true;
+
+ if (!containsMetadataTablet && us.queuedMutations.size() > 0)
+ TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
+
+ Span prep = Trace.start("prep");
+ for (Entry<Tablet,? extends List<Mutation>> entry : us.queuedMutations.entrySet()) {
+
+ Tablet tablet = entry.getKey();
+ List<Mutation> mutations = entry.getValue();
+ if (mutations.size() > 0) {
+ try {
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.mutationArraySize, mutations.size());
+
+ CommitSession commitSession = tablet.prepareMutationsForCommit(us.cenv, mutations);
+ if (commitSession == null) {
+ if (us.currentTablet == tablet) {
+ us.currentTablet = null;
+ }
+ us.failures.put(tablet.getExtent(), us.successfulCommits.get(tablet));
+ } else {
+ sendables.put(commitSession, mutations);
+ mutationCount += mutations.size();
+ }
+
+ } catch (TConstraintViolationException e) {
+ us.violations.add(e.getViolations());
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.constraintViolations, 0);
+
+ if (e.getNonViolators().size() > 0) {
+ // only log and commit mutations if there were some
+ // that did not
+ // violate constraints... this is what
+ // prepareMutationsForCommit()
+ // expects
+ sendables.put(e.getCommitSession(), e.getNonViolators());
+ }
+
+ mutationCount += mutations.size();
+
+ } catch (HoldTimeoutException t) {
+ error = t;
+ log.debug("Giving up on mutations due to a long memory hold time");
+ break;
+ } catch (Throwable t) {
+ error = t;
+ log.error("Unexpected error preparing for commit", error);
+ break;
+ }
+ }
+ }
+ prep.stop();
+
+ Span wal = Trace.start("wal");
+ long pt2 = System.currentTimeMillis();
+ long avgPrepareTime = (long) ((pt2 - pt1) / (double) us.queuedMutations.size());
+ us.prepareTimes.addStat(pt2 - pt1);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.commitPrep, (avgPrepareTime));
+
+ if (error != null) {
+ for (Entry<CommitSession,List<Mutation>> e : sendables.entrySet()) {
+ e.getKey().abortCommit(e.getValue());
+ }
+ throw new RuntimeException(error);
+ }
+ try {
+ while (true) {
+ try {
+ long t1 = System.currentTimeMillis();
+
+ logger.logManyTablets(sendables);
+
+ long t2 = System.currentTimeMillis();
+ us.walogTimes.addStat(t2 - t1);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.waLogWriteTime, (t2 - t1));
+
+ break;
+ } catch (IOException ex) {
+ log.warn("logging mutations failed, retrying");
+ } catch (Throwable t) {
+ log.error("Unknown exception logging mutations, counts for mutations in flight not decremented!", t);
+ throw new RuntimeException(t);
+ }
+ }
+
+ wal.stop();
+
+ Span commit = Trace.start("commit");
+ long t1 = System.currentTimeMillis();
+ for (Entry<CommitSession,? extends List<Mutation>> entry : sendables.entrySet()) {
+ CommitSession commitSession = entry.getKey();
+ List<Mutation> mutations = entry.getValue();
+
+ commitSession.commit(mutations);
+
+ Tablet tablet = commitSession.getTablet();
+
+ if (tablet == us.currentTablet) {
+ // because constraint violations may filter out some
+ // mutations, for proper
+ // accounting with the client code, need to increment
+ // the count based
+ // on the original number of mutations from the client
+ // NOT the filtered number
+ us.successfulCommits.increment(tablet, us.queuedMutations.get(tablet).size());
+ }
+ }
+ long t2 = System.currentTimeMillis();
+
+ long avgCommitTime = (long) ((t2 - t1) / (double) sendables.size());
+
+ us.flushTime += (t2 - pt1);
+ us.commitTimes.addStat(t2 - t1);
+
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.commitTime, avgCommitTime);
+ commit.stop();
+ } finally {
+ us.queuedMutations.clear();
+ if (us.currentTablet != null) {
+ us.queuedMutations.put(us.currentTablet, new ArrayList<Mutation>());
+ }
+ us.queuedMutationSize = 0;
+ }
+ us.totalUpdates += mutationCount;
+ }
+
+ @Override
+ public UpdateErrors closeUpdate(TInfo tinfo, long updateID) throws NoSuchScanIDException {
+ UpdateSession us = (UpdateSession) sessionManager.removeSession(updateID);
+ if (us == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ // clients may or may not see data from an update session while
+ // it is in progress, however when the update session is closed
+ // want to ensure that reads wait for the write to finish
+ long opid = writeTracker.startWrite(us.queuedMutations.keySet());
+
+ try {
+ flush(us);
+ } finally {
+ writeTracker.finishWrite(opid);
+ }
+
+ log.debug(String.format("UpSess %s %,d in %.3fs, at=[%s] ft=%.3fs(pt=%.3fs lt=%.3fs ct=%.3fs)", TServerUtils.clientAddress.get(), us.totalUpdates,
+ (System.currentTimeMillis() - us.startTime) / 1000.0, us.authTimes.toString(), us.flushTime / 1000.0, us.prepareTimes.getSum() / 1000.0,
+ us.walogTimes.getSum() / 1000.0, us.commitTimes.getSum() / 1000.0));
+ if (us.failures.size() > 0) {
+ Entry<KeyExtent,Long> first = us.failures.entrySet().iterator().next();
+ log.debug(String.format("Failures: %d, first extent %s successful commits: %d", us.failures.size(), first.getKey().toString(), first.getValue()));
+ }
+ List<ConstraintViolationSummary> violations = us.violations.asList();
+ if (violations.size() > 0) {
+ ConstraintViolationSummary first = us.violations.asList().iterator().next();
+ log.debug(String.format("Violations: %d, first %s occurs %d", violations.size(), first.violationDescription, first.numberOfViolatingMutations));
+ }
+ if (us.authFailures.size() > 0) {
+ KeyExtent first = us.authFailures.keySet().iterator().next();
+ log.debug(String.format("Authentication Failures: %d, first %s", us.authFailures.size(), first.toString()));
+ }
+
+ return new UpdateErrors(Translator.translate(us.failures, Translators.KET), Translator.translate(violations, Translators.CVST), Translator.translate(
+ us.authFailures, Translators.KET));
+ }
+
+ @Override
+ public void update(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, TMutation tmutation) throws NotServingTabletException,
+ ConstraintViolationException, ThriftSecurityException {
+
+ if (!security.canWrite(credentials, new String(tkeyExtent.getTable(), Constants.UTF8)))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ KeyExtent keyExtent = new KeyExtent(tkeyExtent);
+ Tablet tablet = onlineTablets.get(new KeyExtent(keyExtent));
+ if (tablet == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+
+ if (!keyExtent.isMeta())
+ TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
+
+ long opid = writeTracker.startWrite(TabletType.type(keyExtent));
+
+ try {
+ Mutation mutation = new ServerMutation(tmutation);
+ List<Mutation> mutations = Collections.singletonList(mutation);
+
+ Span prep = Trace.start("prep");
+ CommitSession cs = tablet.prepareMutationsForCommit(new TservConstraintEnv(security, credentials), mutations);
+ prep.stop();
+ if (cs == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+
+ while (true) {
+ try {
+ Span wal = Trace.start("wal");
+ logger.log(cs, cs.getWALogSeq(), mutation);
+ wal.stop();
+ break;
+ } catch (IOException ex) {
+ log.warn(ex, ex);
+ }
+ }
+
+ Span commit = Trace.start("commit");
+ cs.commit(mutations);
+ commit.stop();
+ } catch (TConstraintViolationException e) {
+ throw new ConstraintViolationException(Translator.translate(e.getViolations().asList(), Translators.CVST));
+ } finally {
+ writeTracker.finishWrite(opid);
+ }
+ }
+
+ @Override
+ public void splitTablet(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, ByteBuffer splitPoint)
+ throws NotServingTabletException, ThriftSecurityException {
+
+ String tableId = new String(ByteBufferUtil.toBytes(tkeyExtent.table), Constants.UTF8);
+ if (!security.canSplitTablet(credentials, tableId))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ KeyExtent keyExtent = new KeyExtent(tkeyExtent);
+
+ Tablet tablet = onlineTablets.get(keyExtent);
+ if (tablet == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+
+ if (keyExtent.getEndRow() == null || !keyExtent.getEndRow().equals(ByteBufferUtil.toText(splitPoint))) {
+ try {
+ if (TabletServer.this.splitTablet(tablet, ByteBufferUtil.toBytes(splitPoint)) == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+ } catch (IOException e) {
+ log.warn("Failed to split " + keyExtent, e);
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Override
+ public TabletServerStatus getTabletServerStatus(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ return getStats(sessionManager.getActiveScansPerTable());
+ }
+
+ @Override
+ public List<TabletStats> getTabletStats(TInfo tinfo, TCredentials credentials, String tableId) throws ThriftSecurityException, TException {
+ TreeMap<KeyExtent,Tablet> onlineTabletsCopy;
+ synchronized (onlineTablets) {
+ onlineTabletsCopy = new TreeMap<KeyExtent,Tablet>(onlineTablets);
+ }
+ List<TabletStats> result = new ArrayList<TabletStats>();
+ Text text = new Text(tableId);
+ KeyExtent start = new KeyExtent(text, new Text(), null);
+ for (Entry<KeyExtent,Tablet> entry : onlineTabletsCopy.tailMap(start).entrySet()) {
+ KeyExtent ke = entry.getKey();
+ if (ke.getTableId().compareTo(text) == 0) {
+ Tablet tablet = entry.getValue();
+ TabletStats stats = tablet.timer.getTabletStats();
+ stats.extent = ke.toThrift();
+ stats.ingestRate = tablet.ingestRate();
+ stats.queryRate = tablet.queryRate();
+ stats.splitCreationTime = tablet.getSplitCreationTime();
+ stats.numEntries = tablet.getNumEntries();
+ result.add(stats);
+ }
+ }
+ return result;
+ }
+
+ private ZooCache masterLockCache = new ZooCache();
+
+ private void checkPermission(TCredentials credentials, String lock, boolean requiresSystemPermission, final String request)
+ throws ThriftSecurityException {
+ if (requiresSystemPermission) {
+ boolean fatal = false;
+ try {
+ log.debug("Got " + request + " message from user: " + credentials.getPrincipal());
+ if (!security.canPerformSystemActions(credentials)) {
+ log.warn("Got " + request + " message from user: " + credentials.getPrincipal());
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ }
+ } catch (ThriftSecurityException e) {
+ log.warn("Got " + request + " message from unauthenticatable user: " + e.getUser());
+ if (e.getUser().equals(SecurityConstants.SYSTEM_PRINCIPAL)) {
+ log.fatal("Got message from a service with a mismatched configuration. Please ensure a compatible configuration.", e);
+ fatal = true;
+ }
+ throw e;
+ } finally {
+ if (fatal) {
+ Halt.halt(1, new Runnable() {
+ @Override
+ public void run() {
+ logGCInfo(getSystemConfiguration());
+ }
+ });
+ }
+ }
+ }
+
+ if (tabletServerLock == null || !tabletServerLock.wasLockAcquired()) {
+ log.warn("Got " + request + " message from master before lock acquired, ignoring...");
+ throw new RuntimeException("Lock not acquired");
+ }
+
+ if (tabletServerLock != null && tabletServerLock.wasLockAcquired() && !tabletServerLock.isLocked()) {
+ Halt.halt(1, new Runnable() {
+ @Override
+ public void run() {
+ log.info("Tablet server no longer holds lock during checkPermission() : " + request + ", exiting");
+ logGCInfo(getSystemConfiguration());
+ }
+ });
+ }
+
+ if (lock != null) {
+ ZooUtil.LockID lid = new ZooUtil.LockID(ZooUtil.getRoot(instance) + Constants.ZMASTER_LOCK, lock);
+
+ try {
+ if (!ZooLock.isLockHeld(masterLockCache, lid)) {
+ // maybe the cache is out of date and a new master holds the
+ // lock?
+ masterLockCache.clear();
+ if (!ZooLock.isLockHeld(masterLockCache, lid)) {
+ log.warn("Got " + request + " message from a master that does not hold the current lock " + lock);
+ throw new RuntimeException("bad master lock");
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("bad master lock", e);
+ }
+ }
+ }
+
+ @Override
+ public void loadTablet(TInfo tinfo, TCredentials credentials, String lock, final TKeyExtent textent) {
+
+ try {
+ checkPermission(credentials, lock, true, "loadTablet");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+
+ final KeyExtent extent = new KeyExtent(textent);
+
+ synchronized (unopenedTablets) {
+ synchronized (openingTablets) {
+ synchronized (onlineTablets) {
+
+ // checking if this exact tablet is in any of the sets
+ // below is not a strong enough check
+ // when splits and fix splits occurring
+
+ Set<KeyExtent> unopenedOverlapping = KeyExtent.findOverlapping(extent, unopenedTablets);
+ Set<KeyExtent> openingOverlapping = KeyExtent.findOverlapping(extent, openingTablets);
+ Set<KeyExtent> onlineOverlapping = KeyExtent.findOverlapping(extent, onlineTablets);
+ Set<KeyExtent> all = new HashSet<KeyExtent>();
+ all.addAll(unopenedOverlapping);
+ all.addAll(openingOverlapping);
+ all.addAll(onlineOverlapping);
+
+ if (!all.isEmpty()) {
+ if (all.size() != 1 || !all.contains(extent)) {
+ log.error("Tablet " + extent + " overlaps previously assigned " + unopenedOverlapping + " " + openingOverlapping + " " + onlineOverlapping);
+ }
+ return;
+ }
+
+ unopenedTablets.add(extent);
+ }
+ }
+ }
+
+ // add the assignment job to the appropriate queue
+ log.info("Loading tablet " + extent);
+
+ final Runnable ah = new LoggingRunnable(log, new AssignmentHandler(extent));
+ // Root tablet assignment must take place immediately
+ if (extent.isRootTablet()) {
+ new Daemon("Root Tablet Assignment") {
+ @Override
+ public void run() {
+ ah.run();
+ if (onlineTablets.containsKey(extent)) {
+ log.info("Root tablet loaded: " + extent);
+ } else {
+ log.info("Root tablet failed to load");
+ }
+
+ }
+ }.start();
+ } else {
+ if (extent.isMeta()) {
+ resourceManager.addMetaDataAssignment(ah);
+ } else {
+ resourceManager.addAssignment(ah);
+ }
+ }
+ }
+
+ @Override
+ public void unloadTablet(TInfo tinfo, TCredentials credentials, String lock, TKeyExtent textent, boolean save) {
+ try {
+ checkPermission(credentials, lock, true, "unloadTablet");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+
+ KeyExtent extent = new KeyExtent(textent);
+
+ resourceManager.addMigration(extent, new LoggingRunnable(log, new UnloadTabletHandler(extent, save)));
+ }
+
+ @Override
+ public void flush(TInfo tinfo, TCredentials credentials, String lock, String tableId, ByteBuffer startRow, ByteBuffer endRow) {
+ try {
+ checkPermission(credentials, lock, true, "flush");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+
+ ArrayList<Tablet> tabletsToFlush = new ArrayList<Tablet>();
+
+ KeyExtent ke = new KeyExtent(new Text(tableId), ByteBufferUtil.toText(endRow), ByteBufferUtil.toText(startRow));
+
+ synchronized (onlineTablets) {
+ for (Tablet tablet : onlineTablets.values())
+ if (ke.overlaps(tablet.getExtent()))
+ tabletsToFlush.add(tablet);
+ }
+
+ Long flushID = null;
+
+ for (Tablet tablet : tabletsToFlush) {
+ if (flushID == null) {
+ // read the flush id once from zookeeper instead of reading
+ // it for each tablet
+ try {
+ flushID = tablet.getFlushID();
+ } catch (NoNodeException e) {
+ // table was probably deleted
+ log.info("Asked to flush table that has no flush id " + ke + " " + e.getMessage());
+ return;
+ }
+ }
+ tablet.flush(flushID);
+ }
+ }
+
+ @Override
+ public void flushTablet(TInfo tinfo, TCredentials credentials, String lock, TKeyExtent textent) throws TException {
+ try {
+ checkPermission(credentials, lock, true, "flushTablet");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+
+ Tablet tablet = onlineTablets.get(new KeyExtent(textent));
+ if (tablet != null) {
+ log.info("Flushing " + tablet.getExtent());
+ try {
+ tablet.flush(tablet.getFlushID());
+ } catch (NoNodeException nne) {
+ log.info("Asked to flush tablet that has no flush id " + new KeyExtent(textent) + " " + nne.getMessage());
+ }
+ }
+ }
+
+ @Override
+ public void halt(TInfo tinfo, TCredentials credentials, String lock) throws ThriftSecurityException {
+
+ checkPermission(credentials, lock, true, "halt");
+
+ Halt.halt(0, new Runnable() {
+ @Override
+ public void run() {
+ log.info("Master requested tablet server halt");
+ logGCInfo(getSystemConfiguration());
+ serverStopRequested = true;
+ try {
+ tabletServerLock.unlock();
+ } catch (Exception e) {
+ log.error(e, e);
+ }
+ }
+ });
+ }
+
+ @Override
+ public void fastHalt(TInfo info, TCredentials credentials, String lock) {
+ try {
+ halt(info, credentials, lock);
+ } catch (Exception e) {
+ log.warn("Error halting", e);
+ }
+ }
+
+ @Override
+ public TabletStats getHistoricalStats(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ return statsKeeper.getTabletStats();
+ }
+
+ @Override
+ public List<ActiveScan> getActiveScans(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ try {
+ checkPermission(credentials, null, true, "getScans");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw e;
+ }
+
+ return sessionManager.getActiveScans();
+ }
+
+ @Override
+ public void chop(TInfo tinfo, TCredentials credentials, String lock, TKeyExtent textent) throws TException {
+ try {
+ checkPermission(credentials, lock, true, "chop");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+
+ KeyExtent ke = new KeyExtent(textent);
+
+ Tablet tablet = onlineTablets.get(ke);
+ if (tablet != null) {
+ tablet.chopFiles();
+ }
+ }
+
+ @Override
+ public void compact(TInfo tinfo, TCredentials credentials, String lock, String tableId, ByteBuffer startRow, ByteBuffer endRow)
+ throws TException {
+ try {
+ checkPermission(credentials, lock, true, "compact");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+
+ KeyExtent ke = new KeyExtent(new Text(tableId), ByteBufferUtil.toText(endRow), ByteBufferUtil.toText(startRow));
+
+ ArrayList<Tablet> tabletsToCompact = new ArrayList<Tablet>();
+ synchronized (onlineTablets) {
+ for (Tablet tablet : onlineTablets.values())
+ if (ke.overlaps(tablet.getExtent()))
+ tabletsToCompact.add(tablet);
+ }
+
+ Long compactionId = null;
+
+ for (Tablet tablet : tabletsToCompact) {
+ // all for the same table id, so only need to read
+ // compaction id once
+ if (compactionId == null)
+ try {
+ compactionId = tablet.getCompactionID().getFirst();
+ } catch (NoNodeException e) {
+ log.info("Asked to compact table with no compaction id " + ke + " " + e.getMessage());
+ return;
+ }
+ tablet.compactAll(compactionId);
+ }
+
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface#removeLogs(org.apache.accumulo.trace.thrift.TInfo,
- * org.apache.accumulo.core.security.thrift.Credentials, java.util.List)
- */
+ @Override
+ public void removeLogs(TInfo tinfo, TCredentials credentials, List<String> filenames) throws TException {
+ String myname = getClientAddressString();
+ myname = myname.replace(':', '+');
+ Path logDir = new Path(Constants.getWalDirectory(acuConf), myname);
+ Set<String> loggers = new HashSet<String>();
+ logger.getLoggers(loggers);
+ nextFile: for (String filename : filenames) {
+ for (String logger : loggers) {
+ if (logger.contains(filename))
+ continue nextFile;
+ }
+ List<Tablet> onlineTabletsCopy = new ArrayList<Tablet>();
+ synchronized (onlineTablets) {
+ onlineTabletsCopy.addAll(onlineTablets.values());
+ }
+ for (Tablet tablet : onlineTabletsCopy) {
+ for (String current : tablet.getCurrentLogs()) {
+ if (current.contains(filename)) {
+ log.info("Attempted to delete " + filename + " from tablet " + tablet.getExtent());
+ continue nextFile;
+ }
+ }
+ }
+ try {
+ String source = logDir + "/" + filename;
+ if (acuConf.getBoolean(Property.TSERV_ARCHIVE_WALOGS)) {
+ String walogArchive = Constants.getBaseDir(acuConf) + "/walogArchive";
+ fs.mkdirs(new Path(walogArchive));
+ String dest = walogArchive + "/" + filename;
+ log.info("Archiving walog " + source + " to " + dest);
+ if (!fs.rename(new Path(source), new Path(dest)))
+ log.error("rename is unsuccessful");
+ } else {
+ log.info("Deleting walog " + filename);
+ Trash trash = new Trash(fs, fs.getConf());
+ Path sourcePath = new Path(source);
+ if (!(!acuConf.getBoolean(Property.GC_TRASH_IGNORE) && trash.moveToTrash(sourcePath)) && !fs.delete(sourcePath, true))
+ log.warn("Failed to delete walog " + source);
+ Path recoveryPath = new Path(Constants.getRecoveryDir(acuConf), filename);
+ try {
+ if (trash.moveToTrash(recoveryPath) || fs.delete(recoveryPath, true))
+ log.info("Deleted any recovery log " + filename);
+ } catch (FileNotFoundException ex) {
+ // ignore
+ }
+
+ }
+ } catch (IOException e) {
+ log.warn("Error attempting to delete write-ahead log " + filename + ": " + e);
+ }
+ }
+ }
+
+ @Override
+ public List<ActiveCompaction> getActiveCompactions(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ try {
+ checkPermission(credentials, null, true, "getActiveCompactions");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw e;
+ }
+
+ List<CompactionInfo> compactions = Compactor.getRunningCompactions();
+ List<ActiveCompaction> ret = new ArrayList<ActiveCompaction>(compactions.size());
+
+ for (CompactionInfo compactionInfo : compactions) {
+ ret.add(compactionInfo.toThrift());
+ }
+
+ return ret;
+ }
+ }
+
+ private class SplitRunner implements Runnable {
+ private Tablet tablet;
+
+ public SplitRunner(Tablet tablet) {
+ this.tablet = tablet;
+ }
+
+ @Override
+ public void run() {
+ if (majorCompactorDisabled) {
+ // this will make split task that were queued when shutdown was
+ // initiated exit
+ return;
+ }
+
+ splitTablet(tablet);
+ }
+ }
+
+ boolean isMajorCompactionDisabled() {
+ return majorCompactorDisabled;
+ }
+
+ void executeSplit(Tablet tablet) {
+ resourceManager.executeSplit(tablet.getExtent(), new LoggingRunnable(log, new SplitRunner(tablet)));
+ }
+
+ private class MajorCompactor implements Runnable {
+
+ @Override
+ public void run() {
+ while (!majorCompactorDisabled) {
+ try {
+ UtilWaitThread.sleep(getSystemConfiguration().getTimeInMillis(Property.TSERV_MAJC_DELAY));
+
+ TreeMap<KeyExtent,Tablet> copyOnlineTablets = new TreeMap<KeyExtent,Tablet>();
+
+ synchronized (onlineTablets) {
+ copyOnlineTablets.putAll(onlineTablets); // avoid
+ // concurrent
+ // modification
+ }
+
+ int numMajorCompactions
<TRUNCATED>
[24/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
index f4408f5,0000000..ae5e395
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
@@@ -1,462 -1,0 +1,456 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Level;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class AccumuloInputFormatTest {
+
+ private static final String PREFIX = AccumuloInputFormatTest.class.getSimpleName();
+
+ /**
+ * Test basic setting & getting of max versions.
+ *
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @Deprecated
+ @Test
+ public void testMaxVersions() throws IOException {
+ Job job = new Job();
+ AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 1);
+ int version = AccumuloInputFormat.getMaxVersions(job.getConfiguration());
+ assertEquals(1, version);
+ }
+
+ /**
+ * Test max versions with an invalid value.
+ *
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @Deprecated
+ @Test(expected = IOException.class)
+ public void testMaxVersionsLessThan1() throws IOException {
+ Job job = new Job();
+ AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 0);
+ }
+
+ /**
+ * Test no max version configured.
- *
- * @throws IOException
+ */
+ @Deprecated
+ @Test
+ public void testNoMaxVersion() throws IOException {
+ Job job = new Job();
+ assertEquals(-1, AccumuloInputFormat.getMaxVersions(job.getConfiguration()));
+ }
+
+ /**
+ * Check that the iterator configuration is getting stored in the Job conf correctly.
- *
- * @throws IOException
+ */
+ @Test
+ public void testSetIterator() throws IOException {
+ Job job = new Job();
+
+ IteratorSetting is = new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator");
+ AccumuloInputFormat.addIterator(job, is);
+ Configuration conf = job.getConfiguration();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ is.write(new DataOutputStream(baos));
+ String iterators = conf.get("AccumuloInputFormat.ScanOpts.Iterators");
+ assertEquals(new String(Base64.encodeBase64(baos.toByteArray())), iterators);
+ }
+
+ @Test
+ public void testAddIterator() throws IOException {
+ Job job = new Job();
+
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", WholeRowIterator.class));
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
+ IteratorSetting iter = new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator");
+ iter.addOption("v1", "1");
+ iter.addOption("junk", "\0omg:!\\xyzzy");
+ AccumuloInputFormat.addIterator(job, iter);
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+
+ // Check the list size
+ assertTrue(list.size() == 3);
+
+ // Walk the list and make sure our settings are correct
+ IteratorSetting setting = list.get(0);
+ assertEquals(1, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.user.WholeRowIterator", setting.getIteratorClass());
+ assertEquals("WholeRow", setting.getName());
+ assertEquals(0, setting.getOptions().size());
+
+ setting = list.get(1);
+ assertEquals(2, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
+ assertEquals("Versions", setting.getName());
+ assertEquals(0, setting.getOptions().size());
+
+ setting = list.get(2);
+ assertEquals(3, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
+ assertEquals("Count", setting.getName());
+ assertEquals(2, setting.getOptions().size());
+ assertEquals("1", setting.getOptions().get("v1"));
+ assertEquals("\0omg:!\\xyzzy", setting.getOptions().get("junk"));
+ }
+
+ /**
+ * Test adding iterator options where the keys and values contain both the FIELD_SEPARATOR character (':') and ITERATOR_SEPARATOR (',') characters. There
+ * should be no exceptions thrown when trying to parse these types of option entries.
+ *
+ * This test makes sure that the expected raw values, as appears in the Job, are equal to what's expected.
+ */
+ @Test
+ public void testIteratorOptionEncoding() throws Throwable {
+ String key = "colon:delimited:key";
+ String value = "comma,delimited,value";
+ IteratorSetting someSetting = new IteratorSetting(1, "iterator", "Iterator.class");
+ someSetting.addOption(key, value);
+ Job job = new Job();
+ AccumuloInputFormat.addIterator(job, someSetting);
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+ assertEquals(1, list.size());
+ assertEquals(1, list.get(0).getOptions().size());
+ assertEquals(list.get(0).getOptions().get(key), value);
+
+ someSetting.addOption(key + "2", value);
+ someSetting.setPriority(2);
+ someSetting.setName("it2");
+ AccumuloInputFormat.addIterator(job, someSetting);
+ list = AccumuloInputFormat.getIterators(job);
+ assertEquals(2, list.size());
+ assertEquals(1, list.get(0).getOptions().size());
+ assertEquals(list.get(0).getOptions().get(key), value);
+ assertEquals(2, list.get(1).getOptions().size());
+ assertEquals(list.get(1).getOptions().get(key), value);
+ assertEquals(list.get(1).getOptions().get(key + "2"), value);
+ }
+
+ /**
+ * Test getting iterator settings for multiple iterators set
- *
- * @throws IOException
+ */
+ @Test
+ public void testGetIteratorSettings() throws IOException {
+ Job job = new Job();
+
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator"));
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator"));
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+
+ // Check the list size
+ assertTrue(list.size() == 3);
+
+ // Walk the list and make sure our settings are correct
+ IteratorSetting setting = list.get(0);
+ assertEquals(1, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.WholeRowIterator", setting.getIteratorClass());
+ assertEquals("WholeRow", setting.getName());
+
+ setting = list.get(1);
+ assertEquals(2, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
+ assertEquals("Versions", setting.getName());
+
+ setting = list.get(2);
+ assertEquals(3, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
+ assertEquals("Count", setting.getName());
+
+ }
+
+ @Test
+ public void testSetRegex() throws IOException {
+ Job job = new Job();
+
+ String regex = ">\"*%<>\'\\";
+
+ IteratorSetting is = new IteratorSetting(50, regex, RegExFilter.class);
+ RegExFilter.setRegexs(is, regex, null, null, null, false);
+ AccumuloInputFormat.addIterator(job, is);
+
+ assertTrue(regex.equals(AccumuloInputFormat.getIterators(job).get(0).getName()));
+ }
+
+ private static AssertionError e1 = null;
+ private static AssertionError e2 = null;
+
+ private static class MRTester extends Configured implements Tool {
+ private static class TestMapper extends Mapper<Key,Value,Key,Value> {
+ Key key = null;
+ int count = 0;
+
+ @Override
+ protected void map(Key k, Value v, Context context) throws IOException, InterruptedException {
+ try {
+ if (key != null)
+ assertEquals(key.getRow().toString(), new String(v.get()));
+ assertEquals(k.getRow(), new Text(String.format("%09x", count + 1)));
+ assertEquals(new String(v.get()), String.format("%09x", count));
+ } catch (AssertionError e) {
+ e1 = e;
+ }
+ key = new Key(k);
+ count++;
+ }
+
+ @Override
+ protected void cleanup(Context context) throws IOException, InterruptedException {
+ try {
+ assertEquals(100, count);
+ } catch (AssertionError e) {
+ e2 = e;
+ }
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+
+ if (args.length != 5) {
+ throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <user> <pass> <table> <instanceName> <inputFormatClass>");
+ }
+
+ String user = args[0];
+ String pass = args[1];
+ String table = args[2];
+ String instanceName = args[3];
+ String inputFormatClassName = args[4];
+ @SuppressWarnings("unchecked")
+ Class<? extends InputFormat<?,?>> inputFormatClass = (Class<? extends InputFormat<?,?>>) Class.forName(inputFormatClassName);
+
+ Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+ job.setJarByClass(this.getClass());
+
+ job.setInputFormatClass(inputFormatClass);
+
+ AccumuloInputFormat.setConnectorInfo(job, user, new PasswordToken(pass));
+ AccumuloInputFormat.setInputTableName(job, table);
+ AccumuloInputFormat.setMockInstance(job, instanceName);
+
+ job.setMapperClass(TestMapper.class);
+ job.setMapOutputKeyClass(Key.class);
+ job.setMapOutputValueClass(Value.class);
+ job.setOutputFormatClass(NullOutputFormat.class);
+
+ job.setNumReduceTasks(0);
+
+ job.waitForCompletion(true);
+
+ return job.isSuccessful() ? 0 : 1;
+ }
+
+ public static int main(String[] args) throws Exception {
+ return ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args);
+ }
+ }
+
+ @Test
+ public void testMap() throws Exception {
+ final String INSTANCE_NAME = PREFIX + "_mapreduce_instance";
+ final String TEST_TABLE_1 = PREFIX + "_mapreduce_table_1";
+
+ MockInstance mockInstance = new MockInstance(INSTANCE_NAME);
+ Connector c = mockInstance.getConnector("root", new PasswordToken(""));
+ c.tableOperations().create(TEST_TABLE_1);
+ BatchWriter bw = c.createBatchWriter(TEST_TABLE_1, new BatchWriterConfig());
+ for (int i = 0; i < 100; i++) {
+ Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
+ m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
+ bw.addMutation(m);
+ }
+ bw.close();
+
+ Assert.assertEquals(0, MRTester.main(new String[] {"root", "", TEST_TABLE_1, INSTANCE_NAME, AccumuloInputFormat.class.getCanonicalName()}));
+ assertNull(e1);
+ assertNull(e2);
+ }
+
+ @Test
+ public void testCorrectRangeInputSplits() throws Exception {
+ Job job = new Job(new Configuration(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+
+ String username = "user", table = "table", instance = "instance";
+ PasswordToken password = new PasswordToken("password");
+ Authorizations auths = new Authorizations("foo");
+ Collection<Pair<Text,Text>> fetchColumns = Collections.singleton(new Pair<Text,Text>(new Text("foo"), new Text("bar")));
+ boolean isolated = true, localIters = true;
+ Level level = Level.WARN;
+
+ Instance inst = new MockInstance(instance);
+ Connector connector = inst.getConnector(username, password);
+ connector.tableOperations().create(table);
+
+ AccumuloInputFormat.setConnectorInfo(job, username, password);
+ AccumuloInputFormat.setInputTableName(job, table);
+ AccumuloInputFormat.setScanAuthorizations(job, auths);
+ AccumuloInputFormat.setMockInstance(job, instance);
+ AccumuloInputFormat.setScanIsolation(job, isolated);
+ AccumuloInputFormat.setLocalIterators(job, localIters);
+ AccumuloInputFormat.fetchColumns(job, fetchColumns);
+ AccumuloInputFormat.setLogLevel(job, level);
+
+ AccumuloInputFormat aif = new AccumuloInputFormat();
+
+ List<InputSplit> splits = aif.getSplits(job);
+
+ Assert.assertEquals(1, splits.size());
+
+ InputSplit split = splits.get(0);
+
+ Assert.assertEquals(RangeInputSplit.class, split.getClass());
+
+ RangeInputSplit risplit = (RangeInputSplit) split;
+
+ Assert.assertEquals(username, risplit.getPrincipal());
+ Assert.assertEquals(table, risplit.getTable());
+ Assert.assertEquals(password, risplit.getToken());
+ Assert.assertEquals(auths, risplit.getAuths());
+ Assert.assertEquals(instance, risplit.getInstanceName());
+ Assert.assertEquals(isolated, risplit.isIsolatedScan());
+ Assert.assertEquals(localIters, risplit.usesLocalIterators());
+ Assert.assertEquals(fetchColumns, risplit.getFetchedColumns());
+ Assert.assertEquals(level, risplit.getLogLevel());
+ }
+
+ static class TestMapper extends Mapper<Key,Value,Key,Value> {
+ Key key = null;
+ int count = 0;
+
+ @Override
+ protected void map(Key k, Value v, Context context) throws IOException, InterruptedException {
+ if (key != null)
+ assertEquals(key.getRow().toString(), new String(v.get()));
+ assertEquals(k.getRow(), new Text(String.format("%09x", count + 1)));
+ assertEquals(new String(v.get()), String.format("%09x", count));
+ key = new Key(k);
+ count++;
+ }
+ }
+
+ @Test
+ public void testPartialInputSplitDelegationToConfiguration() throws Exception {
+ String user = "testPartialInputSplitUser";
+ PasswordToken password = new PasswordToken("");
+
+ MockInstance mockInstance = new MockInstance("testPartialInputSplitDelegationToConfiguration");
+ Connector c = mockInstance.getConnector(user, password);
+ c.tableOperations().create("testtable");
+ BatchWriter bw = c.createBatchWriter("testtable", new BatchWriterConfig());
+ for (int i = 0; i < 100; i++) {
+ Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
+ m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
+ bw.addMutation(m);
+ }
+ bw.close();
+
+ Assert.assertEquals(
+ 0,
+ MRTester.main(new String[] {user, "", "testtable", "testPartialInputSplitDelegationToConfiguration",
+ EmptySplitsAccumuloInputFormat.class.getCanonicalName()}));
+ assertNull(e1);
+ assertNull(e2);
+ }
+
+ @Test
+ public void testPartialFailedInputSplitDelegationToConfiguration() throws Exception {
+ String user = "testPartialFailedInputSplit";
+ PasswordToken password = new PasswordToken("");
+
+ MockInstance mockInstance = new MockInstance("testPartialFailedInputSplitDelegationToConfiguration");
+ Connector c = mockInstance.getConnector(user, password);
+ c.tableOperations().create("testtable");
+ BatchWriter bw = c.createBatchWriter("testtable", new BatchWriterConfig());
+ for (int i = 0; i < 100; i++) {
+ Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
+ m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
+ bw.addMutation(m);
+ }
+ bw.close();
+
+ // We should fail before we even get into the Mapper because we can't make the RecordReader
+ Assert.assertEquals(
+ 1,
+ MRTester.main(new String[] {user, "", "testtable", "testPartialFailedInputSplitDelegationToConfiguration",
+ BadPasswordSplitsAccumuloInputFormat.class.getCanonicalName()}));
+ assertNull(e1);
+ assertNull(e2);
+ }
+
+ @Test
+ public void testEmptyColumnFamily() throws IOException {
+ Job job = new Job();
+ Set<Pair<Text,Text>> cols = new HashSet<Pair<Text,Text>>();
+ cols.add(new Pair<Text,Text>(new Text(""), null));
+ cols.add(new Pair<Text,Text>(new Text("foo"), new Text("bar")));
+ cols.add(new Pair<Text,Text>(new Text(""), new Text("bar")));
+ cols.add(new Pair<Text,Text>(new Text(""), new Text("")));
+ cols.add(new Pair<Text,Text>(new Text("foo"), new Text("")));
+ AccumuloInputFormat.fetchColumns(job, cols);
+ Set<Pair<Text,Text>> setCols = AccumuloInputFormat.getFetchedColumns(job);
+ assertEquals(cols, setCols);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/test/java/org/apache/accumulo/core/util/shell/command/FormatterCommandTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/util/shell/command/FormatterCommandTest.java
index 0eb2653,0000000..6000817
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/util/shell/command/FormatterCommandTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/shell/command/FormatterCommandTest.java
@@@ -1,212 -1,0 +1,199 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.util.shell.command;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.util.Iterator;
+import java.util.Map.Entry;
+
- import org.junit.Assert;
-
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.mock.MockShell;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.format.Formatter;
+import org.apache.accumulo.core.util.shell.Shell;
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
++import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Uses the MockShell to test the shell output with Formatters
+ */
+public class FormatterCommandTest {
+ Writer writer = null;
+ InputStream in = null;
+
+ @Test
+ public void test() throws IOException, AccumuloException, AccumuloSecurityException, TableExistsException, ClassNotFoundException {
+ // Keep the Shell AUDIT log off the test output
+ Logger.getLogger(Shell.class).setLevel(Level.WARN);
+
+ final String[] args = new String[] {"--fake", "-u", "root", "-p", ""};
+
+ final String[] commands = createCommands();
+
+ in = MockShell.makeCommands(commands);
+ writer = new StringWriter();
+
+ final MockShell shell = new MockShell(in, writer);
+ shell.config(args);
+
+ // Can't call createtable in the shell with MockAccumulo
+ shell.getConnector().tableOperations().create("test");
+
+ try {
+ shell.start();
+ } catch (Exception e) {
+ Assert.fail("Exception while running commands: " + e.getMessage());
+ }
+
+ shell.getReader().flushConsole();
+
+ final String[] output = StringUtils.split(writer.toString(), '\n');
+
+ boolean formatterOn = false;
+
+ final String[] expectedDefault = new String[] {
+ "row cf:cq [] 1234abcd",
+ "row cf1:cq1 [] 9876fedc",
+ "row2 cf:cq [] 13579bdf",
+ "row2 cf1:cq [] 2468ace"
+ };
+
+ final String[] expectedFormatted = new String[] {
+ "row cf:cq [] 0x31 0x32 0x33 0x34 0x61 0x62 0x63 0x64",
+ "row cf1:cq1 [] 0x39 0x38 0x37 0x36 0x66 0x65 0x64 0x63",
+ "row2 cf:cq [] 0x31 0x33 0x35 0x37 0x39 0x62 0x64 0x66",
+ "row2 cf1:cq [] 0x32 0x34 0x36 0x38 0x61 0x63 0x65"
+ };
+
+ int outputIndex = 0;
+ while (outputIndex < output.length) {
+ final String line = output[outputIndex];
+
+ if (line.startsWith("root@mock-instance")) {
+ if (line.contains("formatter -t test -f org.apache.accumulo.core.util.shell.command.FormatterCommandTest$HexFormatter")) {
+ formatterOn = true;
+ }
+
+ outputIndex++;
+ } else if (line.startsWith("row")) {
+ int expectedIndex = 0;
+ String[] comparisonData;
+
+ // Pick the type of data we expect (formatted or default)
+ if (formatterOn) {
+ comparisonData = expectedFormatted;
+ } else {
+ comparisonData = expectedDefault;
+ }
+
+ // Ensure each output is what we expected
+ while (expectedIndex + outputIndex < output.length && expectedIndex < expectedFormatted.length) {
+ Assert.assertEquals(comparisonData[expectedIndex].trim(), output[expectedIndex + outputIndex].trim());
+ expectedIndex++;
+ }
+
+ outputIndex += expectedIndex;
+ }
+ }
+ }
+
+ private String[] createCommands() {
+ return new String[] {
+ "table test",
+ "insert row cf cq 1234abcd",
+ "insert row cf1 cq1 9876fedc",
+ "insert row2 cf cq 13579bdf",
+ "insert row2 cf1 cq 2468ace",
+ "scan",
+ "formatter -t test -f org.apache.accumulo.core.util.shell.command.FormatterCommandTest$HexFormatter",
+ "scan"
+ };
+ }
+
+ /**
+ * <p>Simple <code>Formatter</code> that will convert each character in the Value
+ * from decimal to hexadecimal. Will automatically skip over characters in the value
+ * which do not fall within the [0-9,a-f] range.</p>
+ *
+ * <p>Example: <code>'0'</code> will be displayed as <code>'0x30'</code></p>
+ */
+ public static class HexFormatter implements Formatter {
+ private Iterator<Entry<Key, Value>> iter = null;
+ private boolean printTs = false;
+
+ private final static String tab = "\t";
+ private final static String newline = "\n";
+
+ public HexFormatter() {}
+
- /* (non-Javadoc)
- * @see java.util.Iterator#hasNext()
- */
+ @Override
+ public boolean hasNext() {
+ return this.iter.hasNext();
+ }
+
- /* (non-Javadoc)
- * @see java.util.Iterator#next()
- */
+ @Override
+ public String next() {
+ final Entry<Key, Value> entry = iter.next();
+
+ String key;
+
+ // Observe the timestamps
+ if (printTs) {
+ key = entry.getKey().toString();
+ } else {
+ key = entry.getKey().toStringNoTime();
+ }
+
+ final Value v = entry.getValue();
+
+ // Approximate how much space we'll need
+ final StringBuilder sb = new StringBuilder(key.length() + v.getSize() * 5);
+
+ sb.append(key).append(tab);
+
+ for (byte b : v.get()) {
+ if ((b >= 48 && b <= 57) || (b >= 97 || b <= 102)) {
+ sb.append(String.format("0x%x ", Integer.valueOf(b)));
+ }
+ }
+
+ sb.append(newline);
+
+ return sb.toString();
+ }
+
- /* (non-Javadoc)
- * @see java.util.Iterator#remove()
- */
+ @Override
+ public void remove() {
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.util.format.Formatter#initialize(java.lang.Iterable, boolean)
- */
+ @Override
+ public void initialize(final Iterable<Entry<Key,Value>> scanner, final boolean printTimestamps) {
+ this.iter = scanner.iterator();
+ this.printTs = printTimestamps;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
index 2ae82b4,0000000..a3bcf62
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
@@@ -1,234 -1,0 +1,229 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.client;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+import java.util.Random;
+
+import org.apache.accumulo.core.cli.BatchScannerOpts;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Internal class used to verify validity of data read.
+ */
+class CountingVerifyingReceiver {
+ private static final Logger log = Logger.getLogger(CountingVerifyingReceiver.class);
+
+ long count = 0;
+ int expectedValueSize = 0;
+ HashMap<Text,Boolean> expectedRows;
+
+ CountingVerifyingReceiver(HashMap<Text,Boolean> expectedRows, int expectedValueSize) {
+ this.expectedRows = expectedRows;
+ this.expectedValueSize = expectedValueSize;
+ }
+
+ public void receive(Key key, Value value) {
+
+ String row = key.getRow().toString();
+ long rowid = Integer.parseInt(row.split("_")[1]);
+
+ byte expectedValue[] = RandomBatchWriter.createValue(rowid, expectedValueSize);
+
+ if (!Arrays.equals(expectedValue, value.get())) {
+ log.error("Got unexpected value for " + key + " expected : " + new String(expectedValue) + " got : " + new String(value.get()));
+ }
+
+ if (!expectedRows.containsKey(key.getRow())) {
+ log.error("Got unexpected key " + key);
+ } else {
+ expectedRows.put(key.getRow(), true);
+ }
+
+ count++;
+ }
+}
+
+/**
+ * Simple example for reading random batches of data from Accumulo. See docs/examples/README.batch for instructions.
+ */
+public class RandomBatchScanner {
+ private static final Logger log = Logger.getLogger(CountingVerifyingReceiver.class);
+
+ /**
+ * Generate a number of ranges, each covering a single random row.
+ *
+ * @param num
+ * the number of ranges to generate
+ * @param min
+ * the minimum row that will be generated
+ * @param max
+ * the maximum row that will be generated
+ * @param r
+ * a random number generator
+ * @param ranges
+ * a set in which to store the generated ranges
+ * @param expectedRows
+ * a map in which to store the rows covered by the ranges (initially mapped to false)
+ */
+ static void generateRandomQueries(int num, long min, long max, Random r, HashSet<Range> ranges, HashMap<Text,Boolean> expectedRows) {
+ log.info(String.format("Generating %,d random queries...", num));
+ while (ranges.size() < num) {
+ long rowid = (Math.abs(r.nextLong()) % (max - min)) + min;
+
+ Text row1 = new Text(String.format("row_%010d", rowid));
+
+ Range range = new Range(new Text(row1));
+ ranges.add(range);
+ expectedRows.put(row1, false);
+ }
+
+ log.info("finished");
+ }
+
+ /**
+ * Prints a count of the number of rows mapped to false.
+ *
+ * @param expectedRows
+ * @return boolean indicating "were all the rows found?"
+ */
+ private static boolean checkAllRowsFound(HashMap<Text,Boolean> expectedRows) {
+ int count = 0;
+ boolean allFound = true;
+ for (Entry<Text,Boolean> entry : expectedRows.entrySet())
+ if (!entry.getValue())
+ count++;
+
+ if (count > 0) {
+ log.warn("Did not find " + count + " rows");
+ allFound = false;
+ }
+ return allFound;
+ }
+
+ /**
+ * Generates a number of random queries, verifies that the key/value pairs returned were in the queried ranges and that the values were generated by
+ * {@link RandomBatchWriter#createValue(long, int)}. Prints information about the results.
+ *
+ * @param num
+ * the number of queries to generate
+ * @param min
+ * the min row to query
+ * @param max
+ * the max row to query
+ * @param evs
+ * the expected size of the values
+ * @param r
+ * a random number generator
+ * @param tsbr
+ * a batch scanner
+ * @return boolean indicating "did the queries go fine?"
+ */
+ static boolean doRandomQueries(int num, long min, long max, int evs, Random r, BatchScanner tsbr) {
+
+ HashSet<Range> ranges = new HashSet<Range>(num);
+ HashMap<Text,Boolean> expectedRows = new java.util.HashMap<Text,Boolean>();
+
+ generateRandomQueries(num, min, max, r, ranges, expectedRows);
+
+ tsbr.setRanges(ranges);
+
+ CountingVerifyingReceiver receiver = new CountingVerifyingReceiver(expectedRows, evs);
+
+ long t1 = System.currentTimeMillis();
+
+ for (Entry<Key,Value> entry : tsbr) {
+ receiver.receive(entry.getKey(), entry.getValue());
+ }
+
+ long t2 = System.currentTimeMillis();
+
+ log.info(String.format("%6.2f lookups/sec %6.2f secs%n", num / ((t2 - t1) / 1000.0), ((t2 - t1) / 1000.0)));
+ log.info(String.format("num results : %,d%n", receiver.count));
+
+ return checkAllRowsFound(expectedRows);
+ }
+
+ public static class Opts extends ClientOnRequiredTable {
+ @Parameter(names="--min", description="miniumum row that will be generated")
+ long min = 0;
+ @Parameter(names="--max", description="maximum ow that will be generated")
+ long max = 0;
+ @Parameter(names="--num", required=true, description="number of ranges to generate")
+ int num = 0;
+ @Parameter(names="--size", required=true, description="size of the value to write")
+ int size = 0;
+ @Parameter(names="--seed", description="seed for pseudo-random number generator")
+ Long seed = null;
+ }
+
+ /**
+ * Scans over a specified number of entries to Accumulo using a {@link BatchScanner}. Completes scans twice to compare times for a fresh query with those for
+ * a repeated query which has cached metadata and connections already established.
- *
- * @param args
- * @throws AccumuloException
- * @throws AccumuloSecurityException
- * @throws TableNotFoundException
+ */
+ public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ Opts opts = new Opts();
+ BatchScannerOpts bsOpts = new BatchScannerOpts();
+ opts.parseArgs(RandomBatchScanner.class.getName(), args, bsOpts);
+
+ Connector connector = opts.getConnector();
+ BatchScanner batchReader = connector.createBatchScanner(opts.tableName, opts.auths, bsOpts.scanThreads);
+ batchReader.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);
+
+ Random r;
+ if (opts.seed == null)
+ r = new Random();
+ else
+ r = new Random(opts.seed);
+
+ // do one cold
+ boolean status = doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader);
+
+ System.gc();
+ System.gc();
+ System.gc();
+
+ if (opts.seed == null)
+ r = new Random();
+ else
+ r = new Random(opts.seed);
+
+ // do one hot (connections already established, metadata table cached)
+ status = status && doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader);
+
+ batchReader.close();
+ if (!status) {
+ System.exit(1);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
index ce91da6,0000000..e76352a
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
@@@ -1,172 -1,0 +1,168 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.client;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.security.SecurityErrorCode;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Simple example for writing random data to Accumulo. See docs/examples/README.batch for instructions.
+ *
+ * The rows of the entries will be randomly generated numbers between a specified min and max (prefixed by "row_"). The column families will be "foo" and column
+ * qualifiers will be "1". The values will be random byte arrays of a specified size.
+ */
+public class RandomBatchWriter {
+
+ /**
+ * Creates a random byte array of specified size using the specified seed.
+ *
+ * @param rowid
+ * the seed to use for the random number generator
+ * @param dataSize
+ * the size of the array
+ * @return a random byte array
+ */
+ public static byte[] createValue(long rowid, int dataSize) {
+ Random r = new Random(rowid);
+ byte value[] = new byte[dataSize];
+
+ r.nextBytes(value);
+
+ // transform to printable chars
+ for (int j = 0; j < value.length; j++) {
+ value[j] = (byte) (((0xff & value[j]) % 92) + ' ');
+ }
+
+ return value;
+ }
+
+ /**
+ * Creates a mutation on a specified row with column family "foo", column qualifier "1", specified visibility, and a random value of specified size.
+ *
+ * @param rowid
+ * the row of the mutation
+ * @param dataSize
+ * the size of the random value
+ * @param visibility
+ * the visibility of the entry to insert
+ * @return a mutation
+ */
+ public static Mutation createMutation(long rowid, int dataSize, ColumnVisibility visibility) {
+ Text row = new Text(String.format("row_%010d", rowid));
+
+ Mutation m = new Mutation(row);
+
+ // create a random value that is a function of the
+ // row id for verification purposes
+ byte value[] = createValue(rowid, dataSize);
+
+ m.put(new Text("foo"), new Text("1"), visibility, new Value(value));
+
+ return m;
+ }
+
+ static class Opts extends ClientOnRequiredTable {
+ @Parameter(names="--num", required=true)
+ int num = 0;
+ @Parameter(names="--min")
+ long min = 0;
+ @Parameter(names="--max")
+ long max = Long.MAX_VALUE;
+ @Parameter(names="--size", required=true, description="size of the value to write")
+ int size = 0;
+ @Parameter(names="--vis", converter=VisibilityConverter.class)
+ ColumnVisibility visiblity = new ColumnVisibility("");
+ @Parameter(names="--seed", description="seed for pseudo-random number generator")
+ Long seed = null;
+ }
+
+ /**
+ * Writes a specified number of entries to Accumulo using a {@link BatchWriter}.
- *
- * @throws AccumuloException
- * @throws AccumuloSecurityException
- * @throws TableNotFoundException
+ */
+ public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ Opts opts = new Opts();
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(RandomBatchWriter.class.getName(), args, bwOpts);
+ if ((opts.max - opts.min) < opts.num) {
+ System.err.println(String.format("You must specify a min and a max that allow for at least num possible values. For example, you requested %d rows, but a min of %d and a max of %d only allows for %d rows.", opts.num, opts.min, opts.max, (opts.max - opts.min)));
+ System.exit(1);
+ }
+ Random r;
+ if (opts.seed == null)
+ r = new Random();
+ else {
+ r = new Random(opts.seed);
+ }
+ Connector connector = opts.getConnector();
+ BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
+
+ // reuse the ColumnVisibility object to improve performance
+ ColumnVisibility cv = opts.visiblity;
+
+ // Generate num unique row ids in the given range
+ HashSet<Long> rowids = new HashSet<Long>(opts.num);
+ while (rowids.size() < opts.num) {
+ rowids.add((Math.abs(r.nextLong()) % (opts.max - opts.min)) + opts.min);
+ }
+ for (long rowid : rowids) {
+ Mutation m = createMutation(rowid, opts.size, cv);
+ bw.addMutation(m);
+ }
+
+ try {
+ bw.close();
+ } catch (MutationsRejectedException e) {
+ if (e.getAuthorizationFailuresMap().size() > 0) {
+ HashMap<String,Set<SecurityErrorCode>> tables = new HashMap<String,Set<SecurityErrorCode>>();
+ for (Entry<KeyExtent,Set<SecurityErrorCode>> ke : e.getAuthorizationFailuresMap().entrySet()) {
+ Set<SecurityErrorCode> secCodes = tables.get(ke.getKey().getTableId().toString());
+ if (secCodes == null) {
+ secCodes = new HashSet<SecurityErrorCode>();
+ tables.put(ke.getKey().getTableId().toString(), secCodes);
+ }
+ secCodes.addAll(ke.getValue());
+ }
+ System.err.println("ERROR : Not authorized to write to tables : " + tables);
+ }
+
+ if (e.getConstraintViolationSummaries().size() > 0) {
+ System.err.println("ERROR : Constraint violations occurred : " + e.getConstraintViolationSummaries());
+ }
+ System.exit(1);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java
index a56fdc0,0000000..c37c1c3
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java
@@@ -1,73 -1,0 +1,68 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.client;
+
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.security.ColumnVisibility;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Simple example for writing random data in sequential order to Accumulo. See docs/examples/README.batch for instructions.
+ */
+public class SequentialBatchWriter {
+
+ static class Opts extends ClientOnRequiredTable {
+ @Parameter(names="--start")
+ long start = 0;
+ @Parameter(names="--num", required=true)
+ long num = 0;
+ @Parameter(names="--size", required=true, description="size of the value to write")
+ int valueSize = 0;
+ @Parameter(names="--vis", converter=VisibilityConverter.class)
+ ColumnVisibility vis = new ColumnVisibility();
+ }
+
+ /**
+ * Writes a specified number of entries to Accumulo using a {@link BatchWriter}. The rows of the entries will be sequential starting at a specified number.
+ * The column families will be "foo" and column qualifiers will be "1". The values will be random byte arrays of a specified size.
- *
- * @throws AccumuloException
- * @throws AccumuloSecurityException
- * @throws TableNotFoundException
- * @throws MutationsRejectedException
+ */
+ public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, MutationsRejectedException {
+ Opts opts = new Opts();
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(SequentialBatchWriter.class.getName(), args, bwOpts);
+ Connector connector = opts.getConnector();
+ BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
+
+ long end = opts.start + opts.num;
+
+ for (long i = opts.start; i < end; i++) {
+ Mutation m = RandomBatchWriter.createMutation(i, opts.valueSize, opts.vis);
+ bw.addMutation(m);
+ }
+
+ bw.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/TraceDumpExample.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/TraceDumpExample.java
index 2947e0e,0000000..70a23e5
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/TraceDumpExample.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/TraceDumpExample.java
@@@ -1,77 -1,0 +1,72 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.client;
+
+import org.apache.accumulo.core.cli.ClientOnDefaultTable;
+import org.apache.accumulo.core.cli.ScannerOpts;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.trace.TraceDump;
+import org.apache.accumulo.core.trace.TraceDump.Printer;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Example of using the TraceDump class to print a formatted view of a Trace
+ *
+ */
+public class TraceDumpExample {
+
+ static class Opts extends ClientOnDefaultTable {
+ public Opts() {
+ super("trace");
+ }
+
+ @Parameter(names = {"--traceid"}, description = "The hex string id of a given trace, for example 16cfbbd7beec4ae3")
+ public String traceId = "";
+ }
+
+ public void dump(Opts opts) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+
+ if (opts.traceId.isEmpty()) {
+ throw new IllegalArgumentException("--traceid option is required");
+ }
+
+ Scanner scanner = opts.getConnector().createScanner(opts.getTableName(), opts.auths);
+ scanner.setRange(new Range(new Text(opts.traceId)));
+ TraceDump.printTrace(scanner, new Printer() {
- public void print(String line) {
++ @Override
++ public void print(String line) {
+ System.out.println(line);
+ }
+ });
+ }
+
- /**
- * @param args
- * @throws AccumuloSecurityException
- * @throws AccumuloException
- * @throws TableNotFoundException
- */
+ public static void main(String[] args) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ TraceDumpExample traceDumpExample = new TraceDumpExample();
+ Opts opts = new Opts();
+ ScannerOpts scannerOpts = new ScannerOpts();
+ opts.parseArgs(TraceDumpExample.class.getName(), args, scannerOpts);
+
+ traceDumpExample.dump(opts);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java
index 744efed,0000000..f11c739
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java
@@@ -1,283 -1,0 +1,280 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.dirlist;
+
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Provides utility methods for getting the info for a file, listing the contents of a directory, and performing single wild card searches on file or directory
+ * names. See docs/examples/README.dirlist for instructions.
+ */
+public class QueryUtil {
+ private Connector conn = null;
+ private String tableName;
+ private Authorizations auths;
+ public static final Text DIR_COLF = new Text("dir");
+ public static final Text FORWARD_PREFIX = new Text("f");
+ public static final Text REVERSE_PREFIX = new Text("r");
+ public static final Text INDEX_COLF = new Text("i");
+ public static final Text COUNTS_COLQ = new Text("counts");
+
+ public QueryUtil(Opts opts) throws AccumuloException,
+ AccumuloSecurityException {
+ conn = opts.getConnector();
+ this.tableName = opts.tableName;
+ this.auths = opts.auths;
+ }
+
+ /**
+ * Calculates the depth of a path, i.e. the number of forward slashes in the path name.
+ *
+ * @param path
+ * the full path of a file or directory
+ * @return the depth of the path
+ */
+ public static int getDepth(String path) {
+ int numSlashes = 0;
+ int index = -1;
+ while ((index = path.indexOf("/", index + 1)) >= 0)
+ numSlashes++;
+ return numSlashes;
+ }
+
+ /**
+ * Given a path, construct an accumulo row prepended with the path's depth for the directory table.
+ *
+ * @param path
+ * the full path of a file or directory
+ * @return the accumulo row associated with this path
+ */
+ public static Text getRow(String path) {
+ Text row = new Text(String.format("%03d", getDepth(path)));
+ row.append(path.getBytes(), 0, path.length());
+ return row;
+ }
+
+ /**
+ * Given a path, construct an accumulo row prepended with the {@link #FORWARD_PREFIX} for the index table.
+ *
+ * @param path
+ * the full path of a file or directory
+ * @return the accumulo row associated with this path
+ */
+ public static Text getForwardIndex(String path) {
+ String part = path.substring(path.lastIndexOf("/") + 1);
+ if (part.length() == 0)
+ return null;
+ Text row = new Text(FORWARD_PREFIX);
+ row.append(part.getBytes(), 0, part.length());
+ return row;
+ }
+
+ /**
+ * Given a path, construct an accumulo row prepended with the {@link #REVERSE_PREFIX} with the path reversed for the index table.
+ *
+ * @param path
+ * the full path of a file or directory
+ * @return the accumulo row associated with this path
+ */
+ public static Text getReverseIndex(String path) {
+ String part = path.substring(path.lastIndexOf("/") + 1);
+ if (part.length() == 0)
+ return null;
+ byte[] rev = new byte[part.length()];
+ int i = part.length() - 1;
+ for (byte b : part.getBytes())
+ rev[i--] = b;
+ Text row = new Text(REVERSE_PREFIX);
+ row.append(rev, 0, rev.length);
+ return row;
+ }
+
+ /**
+ * Returns either the {@link #DIR_COLF} or a decoded string version of the colf.
+ *
+ * @param colf
+ * the column family
+ */
+ public static String getType(Text colf) {
+ if (colf.equals(DIR_COLF))
+ return colf.toString() + ":";
+ return Long.toString(Ingest.encoder.decode(colf.getBytes())) + ":";
+ }
+
+ /**
+ * Scans over the directory table and pulls out stat information about a path.
+ *
+ * @param path
+ * the full path of a file or directory
+ */
+ public Map<String,String> getData(String path) throws TableNotFoundException {
+ if (path.endsWith("/"))
+ path = path.substring(0, path.length() - 1);
+ Scanner scanner = conn.createScanner(tableName, auths);
+ scanner.setRange(new Range(getRow(path)));
+ Map<String,String> data = new TreeMap<String,String>();
+ for (Entry<Key,Value> e : scanner) {
+ String type = getType(e.getKey().getColumnFamily());
+ data.put("fullname", e.getKey().getRow().toString().substring(3));
+ data.put(type + e.getKey().getColumnQualifier().toString() + ":" + e.getKey().getColumnVisibility().toString(), new String(e.getValue().get()));
+ }
+ return data;
+ }
+
+ /**
+ * Uses the directory table to list the contents of a directory.
+ *
+ * @param path
+ * the full path of a directory
+ */
+ public Map<String,Map<String,String>> getDirList(String path) throws TableNotFoundException {
+ if (!path.endsWith("/"))
+ path = path + "/";
+ Map<String,Map<String,String>> fim = new TreeMap<String,Map<String,String>>();
+ Scanner scanner = conn.createScanner(tableName, auths);
+ scanner.setRange(Range.prefix(getRow(path)));
+ for (Entry<Key,Value> e : scanner) {
+ String name = e.getKey().getRow().toString();
+ name = name.substring(name.lastIndexOf("/") + 1);
+ String type = getType(e.getKey().getColumnFamily());
+ if (!fim.containsKey(name)) {
+ fim.put(name, new TreeMap<String,String>());
+ fim.get(name).put("fullname", e.getKey().getRow().toString().substring(3));
+ }
+ fim.get(name).put(type + e.getKey().getColumnQualifier().toString() + ":" + e.getKey().getColumnVisibility().toString(), new String(e.getValue().get()));
+ }
+ return fim;
+ }
+
+ /**
+ * Scans over the index table for files or directories with a given name.
+ *
+ * @param term
+ * the name a file or directory to search for
+ */
+ public Iterable<Entry<Key,Value>> exactTermSearch(String term) throws Exception {
+ System.out.println("executing exactTermSearch for " + term);
+ Scanner scanner = conn.createScanner(tableName, auths);
+ scanner.setRange(new Range(getForwardIndex(term)));
+ return scanner;
+ }
+
+ /**
+ * Scans over the index table for files or directories with a given name, prefix, or suffix (indicated by a wildcard '*' at the beginning or end of the term.
+ *
+ * @param exp
+ * the name a file or directory to search for with an optional wildcard '*' at the beginning or end
+ */
+ public Iterable<Entry<Key,Value>> singleRestrictedWildCardSearch(String exp) throws Exception {
+ if (exp.indexOf("/") >= 0)
+ throw new Exception("this method only works with unqualified names");
+
+ Scanner scanner = conn.createScanner(tableName, auths);
+ if (exp.startsWith("*")) {
+ System.out.println("executing beginning wildcard search for " + exp);
+ exp = exp.substring(1);
+ scanner.setRange(Range.prefix(getReverseIndex(exp)));
+ } else if (exp.endsWith("*")) {
+ System.out.println("executing ending wildcard search for " + exp);
+ exp = exp.substring(0, exp.length() - 1);
+ scanner.setRange(Range.prefix(getForwardIndex(exp)));
+ } else if (exp.indexOf("*") >= 0) {
+ throw new Exception("this method only works for beginning or ending wild cards");
+ } else {
+ return exactTermSearch(exp);
+ }
+ return scanner;
+ }
+
+ /**
+ * Scans over the index table for files or directories with a given name that can contain a single wildcard '*' anywhere in the term.
+ *
+ * @param exp
+ * the name a file or directory to search for with one optional wildcard '*'
+ */
+ public Iterable<Entry<Key,Value>> singleWildCardSearch(String exp) throws Exception {
+ int starIndex = exp.indexOf("*");
+ if (exp.indexOf("*", starIndex + 1) >= 0)
+ throw new Exception("only one wild card for search");
+
+ if (starIndex < 0) {
+ return exactTermSearch(exp);
+ } else if (starIndex == 0 || starIndex == exp.length() - 1) {
+ return singleRestrictedWildCardSearch(exp);
+ }
+
+ String firstPart = exp.substring(0, starIndex);
+ String lastPart = exp.substring(starIndex + 1);
+ String regexString = ".*/" + exp.replace("*", "[^/]*");
+
+ Scanner scanner = conn.createScanner(tableName, auths);
+ if (firstPart.length() >= lastPart.length()) {
+ System.out.println("executing middle wildcard search for " + regexString + " from entries starting with " + firstPart);
+ scanner.setRange(Range.prefix(getForwardIndex(firstPart)));
+ } else {
+ System.out.println("executing middle wildcard search for " + regexString + " from entries ending with " + lastPart);
+ scanner.setRange(Range.prefix(getReverseIndex(lastPart)));
+ }
+ IteratorSetting regex = new IteratorSetting(50, "regex", RegExFilter.class);
+ RegExFilter.setRegexs(regex, null, null, regexString, null, false);
+ scanner.addScanIterator(regex);
+ return scanner;
+ }
+
+ public static class Opts extends ClientOnRequiredTable {
+ @Parameter(names="--path", description="the directory to list")
+ String path = "/";
+ @Parameter(names="--search", description="find a file or directorys with the given name")
+ boolean search = false;
+ }
+
+ /**
+ * Lists the contents of a directory using the directory table, or searches for file or directory names (if the -search flag is included).
- *
- * @param args
- * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(QueryUtil.class.getName(), args);
+ QueryUtil q = new QueryUtil(opts);
+ if (opts.search) {
+ for (Entry<Key,Value> e : q.singleWildCardSearch(opts.path)) {
+ System.out.println(e.getKey().getColumnQualifier());
+ }
+ } else {
+ for (Entry<String,Map<String,String>> e : q.getDirList(opts.path).entrySet()) {
+ System.out.println(e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/NGramIngest.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/NGramIngest.java
index cd6ca40,0000000..dc14512
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/NGramIngest.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/NGramIngest.java
@@@ -1,116 -1,0 +1,113 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.mapreduce;
+
+import java.io.IOException;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Map job to ingest n-gram files from
+ * http://storage.googleapis.com/books/ngrams/books/datasetsv2.html
+ */
+public class NGramIngest extends Configured implements Tool {
+
+ private static final Logger log = Logger.getLogger(NGramIngest.class);
+
+
+ static class Opts extends ClientOnRequiredTable {
+ @Parameter(names = "--input", required=true)
+ String inputDirectory;
+ }
+ static class NGramMapper extends Mapper<LongWritable, Text, Text, Mutation> {
+
+ @Override
+ protected void map(LongWritable location, Text value, Context context) throws IOException, InterruptedException {
+ String parts[] = value.toString().split("\\t");
+ if (parts.length >= 4) {
+ Mutation m = new Mutation(parts[0]);
+ m.put(parts[1], String.format("%010d", Long.parseLong(parts[2])), new Value(parts[3].trim().getBytes()));
+ context.write(null, m);
+ }
+ }
+ }
+
- /**
- * @param args
- */
+ @Override
+ public int run(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(getClass().getName(), args);
+
+ Job job = new Job(getConf(), getClass().getSimpleName());
+ job.setJarByClass(getClass());
+
+ opts.setAccumuloConfigs(job);
+ job.setInputFormatClass(TextInputFormat.class);
+ job.setOutputFormatClass(AccumuloOutputFormat.class);
+
+ job.setMapperClass(NGramMapper.class);
+ job.setMapOutputKeyClass(Text.class);
+ job.setMapOutputValueClass(Mutation.class);
+
+ job.setNumReduceTasks(0);
+ job.setSpeculativeExecution(false);
+
+
+ if (!opts.getConnector().tableOperations().exists(opts.tableName)) {
+ log.info("Creating table " + opts.tableName);
+ opts.getConnector().tableOperations().create(opts.tableName);
+ SortedSet<Text> splits = new TreeSet<Text>();
+ String numbers[] = "1 2 3 4 5 6 7 8 9".split("\\s");
+ String lower[] = "a b c d e f g h i j k l m n o p q r s t u v w x y z".split("\\s");
+ String upper[] = "A B C D E F G H I J K L M N O P Q R S T U V W X Y Z".split("\\s");
+ for (String[] array : new String[][]{numbers, lower, upper}) {
+ for (String s : array) {
+ splits.add(new Text(s));
+ }
+ }
+ opts.getConnector().tableOperations().addSplits(opts.tableName, splits);
+ }
+
+ TextInputFormat.addInputPath(job, new Path(opts.inputDirectory));
+ job.waitForCompletion(true);
+ return job.isSuccessful() ? 0 : 1;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(CachedConfiguration.getInstance(), new NGramIngest(), args);
+ if (res != 0)
+ System.exit(res);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
index d8eedef,0000000..669c76d
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
@@@ -1,130 -1,0 +1,129 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.mapreduce;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Map;
+
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.format.DefaultFormatter;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Takes a table and outputs the specified column to a set of part files on hdfs accumulo accumulo.examples.mapreduce.TableToFile <username> <password>
+ * <tablename> <column> <hdfs-output-path>
+ */
+public class TableToFile extends Configured implements Tool {
+
+ static class Opts extends ClientOnRequiredTable {
+ @Parameter(names = "--output", description = "output directory", required = true)
+ String output;
+ @Parameter(names = "--columns", description = "columns to extract, in cf:cq{,cf:cq,...} form")
+ String columns = "";
+ }
+
+ /**
+ * The Mapper class that given a row number, will generate the appropriate output line.
+ */
+ public static class TTFMapper extends Mapper<Key,Value,NullWritable,Text> {
+ @Override
+ public void map(Key row, Value data, Context context) throws IOException, InterruptedException {
+ final Key r = row;
+ final Value v = data;
+ Map.Entry<Key,Value> entry = new Map.Entry<Key,Value>() {
+ @Override
+ public Key getKey() {
+ return r;
+ }
+
+ @Override
+ public Value getValue() {
+ return v;
+ }
+
+ @Override
+ public Value setValue(Value value) {
+ return null;
+ }
+ };
+ context.write(NullWritable.get(), new Text(DefaultFormatter.formatEntry(entry, false)));
+ context.setStatus("Outputed Value");
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException, AccumuloSecurityException {
+ Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+ job.setJarByClass(this.getClass());
+ Opts opts = new Opts();
+ opts.parseArgs(getClass().getName(), args);
+
+ job.setInputFormatClass(AccumuloInputFormat.class);
+ opts.setAccumuloConfigs(job);
+
+ HashSet<Pair<Text,Text>> columnsToFetch = new HashSet<Pair<Text,Text>>();
+ for (String col : opts.columns.split(",")) {
+ int idx = col.indexOf(":");
+ Text cf = new Text(idx < 0 ? col : col.substring(0, idx));
+ Text cq = idx < 0 ? null : new Text(col.substring(idx + 1));
+ if (cf.getLength() > 0)
+ columnsToFetch.add(new Pair<Text,Text>(cf, cq));
+ }
+ if (!columnsToFetch.isEmpty())
+ AccumuloInputFormat.fetchColumns(job, columnsToFetch);
+
+ job.setMapperClass(TTFMapper.class);
+ job.setMapOutputKeyClass(NullWritable.class);
+ job.setMapOutputValueClass(Text.class);
+
+ job.setNumReduceTasks(0);
+
+ job.setOutputFormatClass(TextOutputFormat.class);
+ TextOutputFormat.setOutputPath(job, new Path(opts.output));
+
+ job.waitForCompletion(true);
+ return job.isSuccessful() ? 0 : 1;
+ }
+
+ /**
+ *
+ * @param args
+ * instanceName zookeepers username password table columns outputpath
- * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(CachedConfiguration.getInstance(), new TableToFile(), args);
+ if (res != 0)
+ System.exit(res);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
index f6d610e,0000000..d98d78b
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
@@@ -1,78 -1,0 +1,75 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.shard;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.cli.BatchScannerOpts;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.IntersectingIterator;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * This program queries a set of terms in the shard table (populated by {@link Index}) using the {@link IntersectingIterator}.
+ *
+ * See docs/examples/README.shard for instructions.
+ */
+
+public class Query {
+
+ static class Opts extends ClientOnRequiredTable {
+ @Parameter(description=" term { <term> ... }")
+ List<String> terms = new ArrayList<String>();
+ }
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ BatchScannerOpts bsOpts = new BatchScannerOpts();
+ opts.parseArgs(Query.class.getName(), args, bsOpts);
+
+ Connector conn = opts.getConnector();
+ BatchScanner bs = conn.createBatchScanner(opts.tableName, opts.auths, bsOpts.scanThreads);
+ bs.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);
+
+ Text columns[] = new Text[opts.terms.size()];
+ int i = 0;
+ for (String term : opts.terms) {
+ columns[i++] = new Text(term);
+ }
+ IteratorSetting ii = new IteratorSetting(20, "ii", IntersectingIterator.class);
+ IntersectingIterator.setColumnFamilies(ii, columns);
+ bs.addScanIterator(ii);
+ bs.setRanges(Collections.singleton(new Range()));
+ for (Entry<Key,Value> entry : bs) {
+ System.out.println(" " + entry.getKey().getColumnQualifier());
+ }
+
+ }
+
+}
[27/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFile.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFile.java
index f2cb326,0000000..b7a927b
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFile.java
@@@ -1,2030 -1,0 +1,1986 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.accumulo.core.file.rfile.bcfile;
+
+import java.io.ByteArrayInputStream;
+import java.io.Closeable;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Comparator;
+
+import org.apache.accumulo.core.file.rfile.bcfile.BCFile.Reader.BlockReader;
+import org.apache.accumulo.core.file.rfile.bcfile.BCFile.Writer.BlockAppender;
+import org.apache.accumulo.core.file.rfile.bcfile.Chunk.ChunkDecoder;
+import org.apache.accumulo.core.file.rfile.bcfile.Chunk.ChunkEncoder;
+import org.apache.accumulo.core.file.rfile.bcfile.CompareUtils.BytesComparator;
+import org.apache.accumulo.core.file.rfile.bcfile.CompareUtils.MemcmpRawComparator;
+import org.apache.accumulo.core.file.rfile.bcfile.Utils.Version;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.RawComparator;
+import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.io.serializer.JavaSerializationComparator;
+
+/**
+ * A TFile is a container of key-value pairs. Both keys and values are type-less bytes. Keys are restricted to 64KB, value length is not restricted (practically
+ * limited to the available disk storage). TFile further provides the following features:
+ * <ul>
+ * <li>Block Compression.
+ * <li>Named meta data blocks.
+ * <li>Sorted or unsorted keys.
+ * <li>Seek by key or by file offset.
+ * </ul>
+ * The memory footprint of a TFile includes the following:
+ * <ul>
+ * <li>Some constant overhead of reading or writing a compressed block.
+ * <ul>
+ * <li>Each compressed block requires one compression/decompression codec for I/O.
+ * <li>Temporary space to buffer the key.
+ * <li>Temporary space to buffer the value (for TFile.Writer only). Values are chunk encoded, so that we buffer at most one chunk of user data. By default, the
+ * chunk buffer is 1MB. Reading chunked value does not require additional memory.
+ * </ul>
+ * <li>TFile index, which is proportional to the total number of Data Blocks. The total amount of memory needed to hold the index can be estimated as
+ * (56+AvgKeySize)*NumBlocks.
+ * <li>MetaBlock index, which is proportional to the total number of Meta Blocks.The total amount of memory needed to hold the index for Meta Blocks can be
+ * estimated as (40+AvgMetaBlockName)*NumMetaBlock.
+ * </ul>
+ * <p>
+ * The behavior of TFile can be customized by the following variables through Configuration:
+ * <ul>
+ * <li><b>tfile.io.chunk.size</b>: Value chunk size. Integer (in bytes). Default to 1MB. Values of the length less than the chunk size is guaranteed to have
+ * known value length in read time (See {@link TFile.Reader.Scanner.Entry#isValueLengthKnown()}).
+ * <li><b>tfile.fs.output.buffer.size</b>: Buffer size used for FSDataOutputStream. Integer (in bytes). Default to 256KB.
+ * <li><b>tfile.fs.input.buffer.size</b>: Buffer size used for FSDataInputStream. Integer (in bytes). Default to 256KB.
+ * </ul>
+ * <p>
+ * Suggestions on performance optimization.
+ * <ul>
+ * <li>Minimum block size. We recommend a setting of minimum block size between 256KB to 1MB for general usage. Larger block size is preferred if files are
+ * primarily for sequential access. However, it would lead to inefficient random access (because there are more data to decompress). Smaller blocks are good for
+ * random access, but require more memory to hold the block index, and may be slower to create (because we must flush the compressor stream at the conclusion of
+ * each data block, which leads to an FS I/O flush). Further, due to the internal caching in Compression codec, the smallest possible block size would be around
+ * 20KB-30KB.
+ * <li>The current implementation does not offer true multi-threading for reading. The implementation uses FSDataInputStream seek()+read(), which is shown to be
+ * much faster than positioned-read call in single thread mode. However, it also means that if multiple threads attempt to access the same TFile (using multiple
+ * scanners) simultaneously, the actual I/O is carried out sequentially even if they access different DFS blocks.
+ * <li>Compression codec. Use "none" if the data is not very compressable (by compressable, I mean a compression ratio at least 2:1). Generally, use "lzo" as
+ * the starting point for experimenting. "gz" overs slightly better compression ratio over "lzo" but requires 4x CPU to compress and 2x CPU to decompress,
+ * comparing to "lzo".
+ * <li>File system buffering, if the underlying FSDataInputStream and FSDataOutputStream is already adequately buffered; or if applications reads/writes keys
+ * and values in large buffers, we can reduce the sizes of input/output buffering in TFile layer by setting the configuration parameters
+ * "tfile.fs.input.buffer.size" and "tfile.fs.output.buffer.size".
+ * </ul>
+ *
+ * Some design rationale behind TFile can be found at <a href=https://issues.apache.org/jira/browse/HADOOP-3315>Hadoop-3315</a>.
+ */
+public class TFile {
+ static final Log LOG = LogFactory.getLog(TFile.class);
+
+ private static final String CHUNK_BUF_SIZE_ATTR = "tfile.io.chunk.size";
+ private static final String FS_INPUT_BUF_SIZE_ATTR = "tfile.fs.input.buffer.size";
+ private static final String FS_OUTPUT_BUF_SIZE_ATTR = "tfile.fs.output.buffer.size";
+
+ static int getChunkBufferSize(Configuration conf) {
+ int ret = conf.getInt(CHUNK_BUF_SIZE_ATTR, 1024 * 1024);
+ return (ret > 0) ? ret : 1024 * 1024;
+ }
+
+ static int getFSInputBufferSize(Configuration conf) {
+ return conf.getInt(FS_INPUT_BUF_SIZE_ATTR, 256 * 1024);
+ }
+
+ static int getFSOutputBufferSize(Configuration conf) {
+ return conf.getInt(FS_OUTPUT_BUF_SIZE_ATTR, 256 * 1024);
+ }
+
+ private static final int MAX_KEY_SIZE = 64 * 1024; // 64KB
+ static final Version API_VERSION = new Version((short) 1, (short) 0);
+
+ /** snappy codec **/
+ public static final String COMPRESSION_SNAPPY = "snappy";
+
+ /** compression: gzip */
+ public static final String COMPRESSION_GZ = "gz";
+ /** compression: lzo */
+ public static final String COMPRESSION_LZO = "lzo";
+ /** compression: none */
+ public static final String COMPRESSION_NONE = "none";
+ /** comparator: memcmp */
+ public static final String COMPARATOR_MEMCMP = "memcmp";
+ /** comparator prefix: java class */
+ public static final String COMPARATOR_JCLASS = "jclass:";
+
+ /**
+ * Make a raw comparator from a string name.
+ *
+ * @param name
+ * Comparator name
+ * @return A RawComparable comparator.
+ */
+ static public Comparator<RawComparable> makeComparator(String name) {
+ return TFileMeta.makeComparator(name);
+ }
+
+ // Prevent the instantiation of TFiles
+ private TFile() {
+ // nothing
+ }
+
+ /**
+ * Get names of supported compression algorithms. The names are acceptable by TFile.Writer.
+ *
+ * @return Array of strings, each represents a supported compression algorithm. Currently, the following compression algorithms are supported.
+ * <ul>
+ * <li>"none" - No compression.
+ * <li>"lzo" - LZO compression.
+ * <li>"gz" - GZIP compression.
+ * <li>"snappy" - Snappy compression
+ * </ul>
+ */
+ public static String[] getSupportedCompressionAlgorithms() {
+ return Compression.getSupportedAlgorithms();
+ }
+
+ /**
+ * TFile Writer.
+ */
+ public static class Writer implements Closeable {
+ // minimum compressed size for a block.
+ private final int sizeMinBlock;
+
+ // Meta blocks.
+ final TFileIndex tfileIndex;
+ final TFileMeta tfileMeta;
+
+ // reference to the underlying BCFile.
+ private BCFile.Writer writerBCF;
+
+ // current data block appender.
+ BlockAppender blkAppender;
+ long blkRecordCount;
+
+ // buffers for caching the key.
+ BoundedByteArrayOutputStream currentKeyBufferOS;
+ BoundedByteArrayOutputStream lastKeyBufferOS;
+
+ // buffer used by chunk codec
+ private byte[] valueBuffer;
+
+ /**
+ * Writer states. The state always transits in circles: READY -> IN_KEY -> END_KEY -> IN_VALUE -> READY.
+ */
+ private enum State {
+ READY, // Ready to start a new key-value pair insertion.
+ IN_KEY, // In the middle of key insertion.
+ END_KEY, // Key insertion complete, ready to insert value.
+ IN_VALUE, // In value insertion.
+ // ERROR, // Error encountered, cannot continue.
+ CLOSED, // TFile already closed.
+ }
+
+ // current state of Writer.
+ State state = State.READY;
+ Configuration conf;
+ long errorCount = 0;
+
+ /**
+ * Constructor
+ *
+ * @param fsdos
+ * output stream for writing. Must be at position 0.
+ * @param minBlockSize
+ * Minimum compressed block size in bytes. A compression block will not be closed until it reaches this size except for the last block.
+ * @param compressName
+ * Name of the compression algorithm. Must be one of the strings returned by {@link TFile#getSupportedCompressionAlgorithms()}.
+ * @param comparator
+ * Leave comparator as null or empty string if TFile is not sorted. Otherwise, provide the string name for the comparison algorithm for keys. Two
+ * kinds of comparators are supported.
+ * <ul>
+ * <li>Algorithmic comparator: binary comparators that is language independent. Currently, only "memcmp" is supported.
+ * <li>Language-specific comparator: binary comparators that can only be constructed in specific language. For Java, the syntax is "jclass:",
+ * followed by the class name of the RawComparator. Currently, we only support RawComparators that can be constructed through the default
+ * constructor (with no parameters). Parameterized RawComparators such as {@link WritableComparator} or {@link JavaSerializationComparator} may not
+ * be directly used. One should write a wrapper class that inherits from such classes and use its default constructor to perform proper
+ * initialization.
+ * </ul>
+ * @param conf
+ * The configuration object.
- * @throws IOException
+ */
+ public Writer(FSDataOutputStream fsdos, int minBlockSize, String compressName, String comparator, Configuration conf) throws IOException {
+ sizeMinBlock = minBlockSize;
+ tfileMeta = new TFileMeta(comparator);
+ tfileIndex = new TFileIndex(tfileMeta.getComparator());
+
+ writerBCF = new BCFile.Writer(fsdos, compressName, conf, true);
+ currentKeyBufferOS = new BoundedByteArrayOutputStream(MAX_KEY_SIZE);
+ lastKeyBufferOS = new BoundedByteArrayOutputStream(MAX_KEY_SIZE);
+ this.conf = conf;
+ }
+
+ /**
+ * Close the Writer. Resources will be released regardless of the exceptions being thrown. Future close calls will have no effect.
+ *
+ * The underlying FSDataOutputStream is not closed.
+ */
+ public void close() throws IOException {
+ if ((state == State.CLOSED)) {
+ return;
+ }
+ try {
+ // First try the normal finish.
+ // Terminate upon the first Exception.
+ if (errorCount == 0) {
+ if (state != State.READY) {
+ throw new IllegalStateException("Cannot close TFile in the middle of key-value insertion.");
+ }
+
+ finishDataBlock(true);
+
+ // first, write out data:TFile.meta
+ BlockAppender outMeta = writerBCF.prepareMetaBlock(TFileMeta.BLOCK_NAME, COMPRESSION_NONE);
+ try {
+ tfileMeta.write(outMeta);
+ } finally {
+ outMeta.close();
+ }
+
+ // second, write out data:TFile.index
+ BlockAppender outIndex = writerBCF.prepareMetaBlock(TFileIndex.BLOCK_NAME);
+ try {
+ tfileIndex.write(outIndex);
+ } finally {
+ outIndex.close();
+ }
+
+ writerBCF.close();
+ }
+ } finally {
+ IOUtils.cleanup(LOG, blkAppender, writerBCF);
+ blkAppender = null;
+ writerBCF = null;
+ state = State.CLOSED;
+ }
+ }
+
+ /**
+ * Adding a new key-value pair to the TFile. This is synonymous to append(key, 0, key.length, value, 0, value.length)
+ *
+ * @param key
+ * Buffer for key.
+ * @param value
+ * Buffer for value.
- * @throws IOException
+ */
+ public void append(byte[] key, byte[] value) throws IOException {
+ append(key, 0, key.length, value, 0, value.length);
+ }
+
+ /**
+ * Adding a new key-value pair to TFile.
+ *
+ * @param key
+ * buffer for key.
+ * @param koff
+ * offset in key buffer.
+ * @param klen
+ * length of key.
+ * @param value
+ * buffer for value.
+ * @param voff
+ * offset in value buffer.
+ * @param vlen
+ * length of value.
+ * @throws IOException
+ * Upon IO errors.
+ * <p>
+ * If an exception is thrown, the TFile will be in an inconsistent state. The only legitimate call after that would be close
+ */
+ public void append(byte[] key, int koff, int klen, byte[] value, int voff, int vlen) throws IOException {
+ if ((koff | klen | (koff + klen) | (key.length - (koff + klen))) < 0) {
+ throw new IndexOutOfBoundsException("Bad key buffer offset-length combination.");
+ }
+
+ if ((voff | vlen | (voff + vlen) | (value.length - (voff + vlen))) < 0) {
+ throw new IndexOutOfBoundsException("Bad value buffer offset-length combination.");
+ }
+
+ try {
+ DataOutputStream dosKey = prepareAppendKey(klen);
+ try {
+ ++errorCount;
+ dosKey.write(key, koff, klen);
+ --errorCount;
+ } finally {
+ dosKey.close();
+ }
+
+ DataOutputStream dosValue = prepareAppendValue(vlen);
+ try {
+ ++errorCount;
+ dosValue.write(value, voff, vlen);
+ --errorCount;
+ } finally {
+ dosValue.close();
+ }
+ } finally {
+ state = State.READY;
+ }
+ }
+
+ /**
+ * Helper class to register key after close call on key append stream.
+ */
+ private class KeyRegister extends DataOutputStream {
+ private final int expectedLength;
+ private boolean closed = false;
+
+ public KeyRegister(int len) {
+ super(currentKeyBufferOS);
+ if (len >= 0) {
+ currentKeyBufferOS.reset(len);
+ } else {
+ currentKeyBufferOS.reset();
+ }
+ expectedLength = len;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed == true) {
+ return;
+ }
+
+ try {
+ ++errorCount;
+ byte[] key = currentKeyBufferOS.getBuffer();
+ int len = currentKeyBufferOS.size();
+ /**
+ * verify length.
+ */
+ if (expectedLength >= 0 && expectedLength != len) {
+ throw new IOException("Incorrect key length: expected=" + expectedLength + " actual=" + len);
+ }
+
+ Utils.writeVInt(blkAppender, len);
+ blkAppender.write(key, 0, len);
+ if (tfileIndex.getFirstKey() == null) {
+ tfileIndex.setFirstKey(key, 0, len);
+ }
+
+ if (tfileMeta.isSorted()) {
+ byte[] lastKey = lastKeyBufferOS.getBuffer();
+ int lastLen = lastKeyBufferOS.size();
+ if (tfileMeta.getComparator().compare(key, 0, len, lastKey, 0, lastLen) < 0) {
+ throw new IOException("Keys are not added in sorted order");
+ }
+ }
+
+ BoundedByteArrayOutputStream tmp = currentKeyBufferOS;
+ currentKeyBufferOS = lastKeyBufferOS;
+ lastKeyBufferOS = tmp;
+ --errorCount;
+ } finally {
+ closed = true;
+ state = State.END_KEY;
+ }
+ }
+ }
+
+ /**
+ * Helper class to register value after close call on value append stream.
+ */
+ private class ValueRegister extends DataOutputStream {
+ private boolean closed = false;
+
+ public ValueRegister(OutputStream os) {
+ super(os);
+ }
+
+ // Avoiding flushing call to down stream.
+ @Override
+ public void flush() {
+ // do nothing
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed == true) {
+ return;
+ }
+
+ try {
+ ++errorCount;
+ super.close();
+ blkRecordCount++;
+ // bump up the total record count in the whole file
+ tfileMeta.incRecordCount();
+ finishDataBlock(false);
+ --errorCount;
+ } finally {
+ closed = true;
+ state = State.READY;
+ }
+ }
+ }
+
+ /**
+ * Obtain an output stream for writing a key into TFile. This may only be called when there is no active Key appending stream or value appending stream.
+ *
+ * @param length
+ * The expected length of the key. If length of the key is not known, set length = -1. Otherwise, the application must write exactly as many bytes
+ * as specified here before calling close on the returned output stream.
+ * @return The key appending output stream.
- * @throws IOException
+ *
+ */
+ public DataOutputStream prepareAppendKey(int length) throws IOException {
+ if (state != State.READY) {
+ throw new IllegalStateException("Incorrect state to start a new key: " + state.name());
+ }
+
+ initDataBlock();
+ DataOutputStream ret = new KeyRegister(length);
+ state = State.IN_KEY;
+ return ret;
+ }
+
+ /**
+ * Obtain an output stream for writing a value into TFile. This may only be called right after a key appending operation (the key append stream must be
+ * closed).
+ *
+ * @param length
+ * The expected length of the value. If length of the value is not known, set length = -1. Otherwise, the application must write exactly as many
+ * bytes as specified here before calling close on the returned output stream. Advertising the value size up-front guarantees that the value is
+ * encoded in one chunk, and avoids intermediate chunk buffering.
- * @throws IOException
- *
+ */
+ public DataOutputStream prepareAppendValue(int length) throws IOException {
+ if (state != State.END_KEY) {
+ throw new IllegalStateException("Incorrect state to start a new value: " + state.name());
+ }
+
+ DataOutputStream ret;
+
+ // unknown length
+ if (length < 0) {
+ if (valueBuffer == null) {
+ valueBuffer = new byte[getChunkBufferSize(conf)];
+ }
+ ret = new ValueRegister(new ChunkEncoder(blkAppender, valueBuffer));
+ } else {
+ ret = new ValueRegister(new Chunk.SingleChunkEncoder(blkAppender, length));
+ }
+
+ state = State.IN_VALUE;
+ return ret;
+ }
+
+ /**
+ * Obtain an output stream for creating a meta block. This function may not be called when there is a key append stream or value append stream active. No
+ * more key-value insertion is allowed after a meta data block has been added to TFile.
+ *
+ * @param name
+ * Name of the meta block.
+ * @param compressName
+ * Name of the compression algorithm to be used. Must be one of the strings returned by {@link TFile#getSupportedCompressionAlgorithms()}.
+ * @return A DataOutputStream that can be used to write Meta Block data. Closing the stream would signal the ending of the block.
- * @throws IOException
+ * @throws MetaBlockAlreadyExists
+ * the Meta Block with the same name already exists.
+ */
+ public DataOutputStream prepareMetaBlock(String name, String compressName) throws IOException, MetaBlockAlreadyExists {
+ if (state != State.READY) {
+ throw new IllegalStateException("Incorrect state to start a Meta Block: " + state.name());
+ }
+
+ finishDataBlock(true);
+ DataOutputStream outputStream = writerBCF.prepareMetaBlock(name, compressName);
+ return outputStream;
+ }
+
+ /**
+ * Obtain an output stream for creating a meta block. This function may not be called when there is a key append stream or value append stream active. No
+ * more key-value insertion is allowed after a meta data block has been added to TFile. Data will be compressed using the default compressor as defined in
+ * Writer's constructor.
+ *
+ * @param name
+ * Name of the meta block.
+ * @return A DataOutputStream that can be used to write Meta Block data. Closing the stream would signal the ending of the block.
- * @throws IOException
+ * @throws MetaBlockAlreadyExists
+ * the Meta Block with the same name already exists.
+ */
+ public DataOutputStream prepareMetaBlock(String name) throws IOException, MetaBlockAlreadyExists {
+ if (state != State.READY) {
+ throw new IllegalStateException("Incorrect state to start a Meta Block: " + state.name());
+ }
+
+ finishDataBlock(true);
+ return writerBCF.prepareMetaBlock(name);
+ }
+
+ /**
+ * Check if we need to start a new data block.
+ *
+ * @throws IOException
+ */
+ private void initDataBlock() throws IOException {
+ // for each new block, get a new appender
+ if (blkAppender == null) {
+ blkAppender = writerBCF.prepareDataBlock();
+ }
+ }
+
+ /**
+ * Close the current data block if necessary.
+ *
+ * @param bForceFinish
+ * Force the closure regardless of the block size.
+ * @throws IOException
+ */
+ void finishDataBlock(boolean bForceFinish) throws IOException {
+ if (blkAppender == null) {
+ return;
+ }
+
+ // exceeded the size limit, do the compression and finish the block
+ if (bForceFinish || blkAppender.getCompressedSize() >= sizeMinBlock) {
+ // keep tracks of the last key of each data block, no padding
+ // for now
+ TFileIndexEntry keyLast = new TFileIndexEntry(lastKeyBufferOS.getBuffer(), 0, lastKeyBufferOS.size(), blkRecordCount);
+ tfileIndex.addEntry(keyLast);
+ // close the appender
+ blkAppender.close();
+ blkAppender = null;
+ blkRecordCount = 0;
+ }
+ }
+ }
+
+ /**
+ * TFile Reader. Users may only read TFiles by creating TFile.Reader.Scanner. objects. A scanner may scan the whole TFile ({@link Reader#createScanner()} ) ,
+ * a portion of TFile based on byte offsets ( {@link Reader#createScanner(long, long)}), or a portion of TFile with keys fall in a certain key range (for
+ * sorted TFile only, {@link Reader#createScanner(byte[], byte[])} or {@link Reader#createScanner(RawComparable, RawComparable)}).
+ */
+ public static class Reader implements Closeable {
+ // The underlying BCFile reader.
+ final BCFile.Reader readerBCF;
+
+ // TFile index, it is loaded lazily.
+ TFileIndex tfileIndex = null;
+ final TFileMeta tfileMeta;
+ final BytesComparator comparator;
+
+ // global begin and end locations.
+ private final Location begin;
+ private final Location end;
+
+ /**
+ * Location representing a virtual position in the TFile.
+ */
+ static final class Location implements Comparable<Location>, Cloneable {
+ private int blockIndex;
+ // distance/offset from the beginning of the block
+ private long recordIndex;
+
+ Location(int blockIndex, long recordIndex) {
+ set(blockIndex, recordIndex);
+ }
+
+ void incRecordIndex() {
+ ++recordIndex;
+ }
+
+ Location(Location other) {
+ set(other);
+ }
+
+ int getBlockIndex() {
+ return blockIndex;
+ }
+
+ long getRecordIndex() {
+ return recordIndex;
+ }
+
+ void set(int blockIndex, long recordIndex) {
+ if ((blockIndex | recordIndex) < 0) {
+ throw new IllegalArgumentException("Illegal parameter for BlockLocation.");
+ }
+ this.blockIndex = blockIndex;
+ this.recordIndex = recordIndex;
+ }
+
+ void set(Location other) {
+ set(other.blockIndex, other.recordIndex);
+ }
+
+ /**
+ * @see java.lang.Comparable#compareTo(java.lang.Object)
+ */
+ @Override
+ public int compareTo(Location other) {
+ return compareTo(other.blockIndex, other.recordIndex);
+ }
+
+ int compareTo(int bid, long rid) {
+ if (this.blockIndex == bid) {
+ long ret = this.recordIndex - rid;
+ if (ret > 0)
+ return 1;
+ if (ret < 0)
+ return -1;
+ return 0;
+ }
+ return this.blockIndex - bid;
+ }
+
+ /**
+ * @see java.lang.Object#clone()
+ */
+ @Override
+ protected Location clone() {
+ return new Location(blockIndex, recordIndex);
+ }
+
+ /**
+ * @see java.lang.Object#hashCode()
+ */
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = prime + blockIndex;
+ result = (int) (prime * result + recordIndex);
+ return result;
+ }
+
+ /**
+ * @see java.lang.Object#equals(java.lang.Object)
+ */
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ Location other = (Location) obj;
+ if (blockIndex != other.blockIndex)
+ return false;
+ if (recordIndex != other.recordIndex)
+ return false;
+ return true;
+ }
+ }
+
+ /**
+ * Constructor
+ *
+ * @param fsdis
+ * FS input stream of the TFile.
+ * @param fileLength
+ * The length of TFile. This is required because we have no easy way of knowing the actual size of the input file through the File input stream.
- * @param conf
- * @throws IOException
+ */
+ public Reader(FSDataInputStream fsdis, long fileLength, Configuration conf) throws IOException {
+ readerBCF = new BCFile.Reader(fsdis, fileLength, conf);
+
+ // first, read TFile meta
+ BlockReader brMeta = readerBCF.getMetaBlock(TFileMeta.BLOCK_NAME);
+ try {
+ tfileMeta = new TFileMeta(brMeta);
+ } finally {
+ brMeta.close();
+ }
+
+ comparator = tfileMeta.getComparator();
+ // Set begin and end locations.
+ begin = new Location(0, 0);
+ end = new Location(readerBCF.getBlockCount(), 0);
+ }
+
+ /**
+ * Close the reader. The state of the Reader object is undefined after close. Calling close() for multiple times has no effect.
+ */
+ public void close() throws IOException {
+ readerBCF.close();
+ }
+
+ /**
+ * Get the begin location of the TFile.
+ *
+ * @return If TFile is not empty, the location of the first key-value pair. Otherwise, it returns end().
+ */
+ Location begin() {
+ return begin;
+ }
+
+ /**
+ * Get the end location of the TFile.
+ *
+ * @return The location right after the last key-value pair in TFile.
+ */
+ Location end() {
+ return end;
+ }
+
+ /**
+ * Get the string representation of the comparator.
+ *
+ * @return If the TFile is not sorted by keys, an empty string will be returned. Otherwise, the actual comparator string that is provided during the TFile
+ * creation time will be returned.
+ */
+ public String getComparatorName() {
+ return tfileMeta.getComparatorString();
+ }
+
+ /**
+ * Is the TFile sorted?
+ *
+ * @return true if TFile is sorted.
+ */
+ public boolean isSorted() {
+ return tfileMeta.isSorted();
+ }
+
+ /**
+ * Get the number of key-value pair entries in TFile.
+ *
+ * @return the number of key-value pairs in TFile
+ */
+ public long getEntryCount() {
+ return tfileMeta.getRecordCount();
+ }
+
+ /**
+ * Lazily loading the TFile index.
+ *
+ * @throws IOException
+ */
+ synchronized void checkTFileDataIndex() throws IOException {
+ if (tfileIndex == null) {
+ BlockReader brIndex = readerBCF.getMetaBlock(TFileIndex.BLOCK_NAME);
+ try {
+ tfileIndex = new TFileIndex(readerBCF.getBlockCount(), brIndex, tfileMeta.getComparator());
+ } finally {
+ brIndex.close();
+ }
+ }
+ }
+
+ /**
+ * Get the first key in the TFile.
+ *
+ * @return The first key in the TFile.
- * @throws IOException
+ */
+ public RawComparable getFirstKey() throws IOException {
+ checkTFileDataIndex();
+ return tfileIndex.getFirstKey();
+ }
+
+ /**
+ * Get the last key in the TFile.
+ *
+ * @return The last key in the TFile.
- * @throws IOException
+ */
+ public RawComparable getLastKey() throws IOException {
+ checkTFileDataIndex();
+ return tfileIndex.getLastKey();
+ }
+
+ /**
+ * Get a Comparator object to compare Entries. It is useful when you want stores the entries in a collection (such as PriorityQueue) and perform sorting or
+ * comparison among entries based on the keys without copying out the key.
+ *
+ * @return An Entry Comparator..
+ */
+ public Comparator<Scanner.Entry> getEntryComparator() {
+ if (!isSorted()) {
+ throw new RuntimeException("Entries are not comparable for unsorted TFiles");
+ }
+
+ return new Comparator<Scanner.Entry>() {
+ /**
+ * Provide a customized comparator for Entries. This is useful if we have a collection of Entry objects. However, if the Entry objects come from
+ * different TFiles, users must ensure that those TFiles share the same RawComparator.
+ */
+ @Override
+ public int compare(Scanner.Entry o1, Scanner.Entry o2) {
+ return comparator.compare(o1.getKeyBuffer(), 0, o1.getKeyLength(), o2.getKeyBuffer(), 0, o2.getKeyLength());
+ }
+ };
+ }
+
+ /**
+ * Get an instance of the RawComparator that is constructed based on the string comparator representation.
+ *
+ * @return a Comparator that can compare RawComparable's.
+ */
+ public Comparator<RawComparable> getComparator() {
+ return comparator;
+ }
+
+ /**
+ * Stream access to a meta block.``
+ *
+ * @param name
+ * The name of the meta block.
+ * @return The input stream.
+ * @throws IOException
+ * on I/O error.
+ * @throws MetaBlockDoesNotExist
+ * If the meta block with the name does not exist.
+ */
+ public DataInputStream getMetaBlock(String name) throws IOException, MetaBlockDoesNotExist {
+ return readerBCF.getMetaBlock(name);
+ }
+
+ /**
+ * if greater is true then returns the beginning location of the block containing the key strictly greater than input key. if greater is false then returns
+ * the beginning location of the block greater than equal to the input key
+ *
+ * @param key
+ * the input key
+ * @param greater
+ * boolean flag
+ * @throws IOException
+ */
+ Location getBlockContainsKey(RawComparable key, boolean greater) throws IOException {
+ if (!isSorted()) {
+ throw new RuntimeException("Seeking in unsorted TFile");
+ }
+ checkTFileDataIndex();
+ int blkIndex = (greater) ? tfileIndex.upperBound(key) : tfileIndex.lowerBound(key);
+ if (blkIndex < 0)
+ return end;
+ return new Location(blkIndex, 0);
+ }
+
+ int compareKeys(byte[] a, int o1, int l1, byte[] b, int o2, int l2) {
+ if (!isSorted()) {
+ throw new RuntimeException("Cannot compare keys for unsorted TFiles.");
+ }
+ return comparator.compare(a, o1, l1, b, o2, l2);
+ }
+
+ int compareKeys(RawComparable a, RawComparable b) {
+ if (!isSorted()) {
+ throw new RuntimeException("Cannot compare keys for unsorted TFiles.");
+ }
+ return comparator.compare(a, b);
+ }
+
+ /**
+ * Get the location pointing to the beginning of the first key-value pair in a compressed block whose byte offset in the TFile is greater than or equal to
+ * the specified offset.
+ *
+ * @param offset
+ * the user supplied offset.
+ * @return the location to the corresponding entry; or end() if no such entry exists.
+ */
+ Location getLocationNear(long offset) {
+ int blockIndex = readerBCF.getBlockIndexNear(offset);
+ if (blockIndex == -1)
+ return end;
+ return new Location(blockIndex, 0);
+ }
+
+ /**
+ * Get a sample key that is within a block whose starting offset is greater than or equal to the specified offset.
+ *
+ * @param offset
+ * The file offset.
+ * @return the key that fits the requirement; or null if no such key exists (which could happen if the offset is close to the end of the TFile).
- * @throws IOException
+ */
+ public RawComparable getKeyNear(long offset) throws IOException {
+ int blockIndex = readerBCF.getBlockIndexNear(offset);
+ if (blockIndex == -1)
+ return null;
+ checkTFileDataIndex();
+ return new ByteArray(tfileIndex.getEntry(blockIndex).key);
+ }
+
+ /**
+ * Get a scanner than can scan the whole TFile.
+ *
+ * @return The scanner object. A valid Scanner is always returned even if the TFile is empty.
- * @throws IOException
+ */
+ public Scanner createScanner() throws IOException {
+ return new Scanner(this, begin, end);
+ }
+
+ /**
+ * Get a scanner that covers a portion of TFile based on byte offsets.
+ *
+ * @param offset
+ * The beginning byte offset in the TFile.
+ * @param length
+ * The length of the region.
+ * @return The actual coverage of the returned scanner tries to match the specified byte-region but always round up to the compression block boundaries. It
+ * is possible that the returned scanner contains zero key-value pairs even if length is positive.
- * @throws IOException
+ */
+ public Scanner createScanner(long offset, long length) throws IOException {
+ return new Scanner(this, offset, offset + length);
+ }
+
+ /**
+ * Get a scanner that covers a portion of TFile based on keys.
+ *
+ * @param beginKey
+ * Begin key of the scan (inclusive). If null, scan from the first key-value entry of the TFile.
+ * @param endKey
+ * End key of the scan (exclusive). If null, scan up to the last key-value entry of the TFile.
+ * @return The actual coverage of the returned scanner will cover all keys greater than or equal to the beginKey and less than the endKey.
- * @throws IOException
+ */
+ public Scanner createScanner(byte[] beginKey, byte[] endKey) throws IOException {
+ return createScanner((beginKey == null) ? null : new ByteArray(beginKey, 0, beginKey.length), (endKey == null) ? null : new ByteArray(endKey, 0,
+ endKey.length));
+ }
+
+ /**
+ * Get a scanner that covers a specific key range.
+ *
+ * @param beginKey
+ * Begin key of the scan (inclusive). If null, scan from the first key-value entry of the TFile.
+ * @param endKey
+ * End key of the scan (exclusive). If null, scan up to the last key-value entry of the TFile.
+ * @return The actual coverage of the returned scanner will cover all keys greater than or equal to the beginKey and less than the endKey.
- * @throws IOException
+ */
+ public Scanner createScanner(RawComparable beginKey, RawComparable endKey) throws IOException {
+ if ((beginKey != null) && (endKey != null) && (compareKeys(beginKey, endKey) >= 0)) {
+ return new Scanner(this, beginKey, beginKey);
+ }
+ return new Scanner(this, beginKey, endKey);
+ }
+
+ /**
+ * The TFile Scanner. The Scanner has an implicit cursor, which, upon creation, points to the first key-value pair in the scan range. If the scan range is
+ * empty, the cursor will point to the end of the scan range.
+ * <p>
+ * Use {@link Scanner#atEnd()} to test whether the cursor is at the end location of the scanner.
+ * <p>
+ * Use {@link Scanner#advance()} to move the cursor to the next key-value pair (or end if none exists). Use seekTo methods ( {@link Scanner#seekTo(byte[])}
+ * or {@link Scanner#seekTo(byte[], int, int)}) to seek to any arbitrary location in the covered range (including backward seeking). Use
+ * {@link Scanner#rewind()} to seek back to the beginning of the scanner. Use {@link Scanner#seekToEnd()} to seek to the end of the scanner.
+ * <p>
+ * Actual keys and values may be obtained through {@link Scanner.Entry} object, which is obtained through {@link Scanner#entry()}.
+ */
+ public static class Scanner implements Closeable {
+ // The underlying TFile reader.
+ final Reader reader;
+ // current block (null if reaching end)
+ private BlockReader blkReader;
+
+ Location beginLocation;
+ Location endLocation;
+ Location currentLocation;
+
+ // flag to ensure value is only examined once.
+ boolean valueChecked = false;
+ // reusable buffer for keys.
+ final byte[] keyBuffer;
+ // length of key, -1 means key is invalid.
+ int klen = -1;
+
+ static final int MAX_VAL_TRANSFER_BUF_SIZE = 128 * 1024;
+ BytesWritable valTransferBuffer;
+
+ DataInputBuffer keyDataInputStream;
+ ChunkDecoder valueBufferInputStream;
+ DataInputStream valueDataInputStream;
+ // vlen == -1 if unknown.
+ int vlen;
+
+ /**
+ * Constructor
+ *
+ * @param reader
+ * The TFile reader object.
+ * @param offBegin
+ * Begin byte-offset of the scan.
+ * @param offEnd
+ * End byte-offset of the scan.
+ * @throws IOException
+ *
+ * The offsets will be rounded to the beginning of a compressed block whose offset is greater than or equal to the specified offset.
+ */
+ protected Scanner(Reader reader, long offBegin, long offEnd) throws IOException {
+ this(reader, reader.getLocationNear(offBegin), reader.getLocationNear(offEnd));
+ }
+
+ /**
+ * Constructor
+ *
+ * @param reader
+ * The TFile reader object.
+ * @param begin
+ * Begin location of the scan.
+ * @param end
+ * End location of the scan.
+ * @throws IOException
+ */
+ Scanner(Reader reader, Location begin, Location end) throws IOException {
+ this.reader = reader;
+ // ensure the TFile index is loaded throughout the life of scanner.
+ reader.checkTFileDataIndex();
+ beginLocation = begin;
+ endLocation = end;
+
+ valTransferBuffer = new BytesWritable();
+ keyBuffer = new byte[MAX_KEY_SIZE];
+ keyDataInputStream = new DataInputBuffer();
+ valueBufferInputStream = new ChunkDecoder();
+ valueDataInputStream = new DataInputStream(valueBufferInputStream);
+
+ if (beginLocation.compareTo(endLocation) >= 0) {
+ currentLocation = new Location(endLocation);
+ } else {
+ currentLocation = new Location(0, 0);
+ initBlock(beginLocation.getBlockIndex());
+ inBlockAdvance(beginLocation.getRecordIndex());
+ }
+ }
+
+ /**
+ * Constructor
+ *
+ * @param reader
+ * The TFile reader object.
+ * @param beginKey
+ * Begin key of the scan. If null, scan from the first <K,V> entry of the TFile.
+ * @param endKey
+ * End key of the scan. If null, scan up to the last <K, V> entry of the TFile.
- * @throws IOException
+ */
+ protected Scanner(Reader reader, RawComparable beginKey, RawComparable endKey) throws IOException {
+ this(reader, (beginKey == null) ? reader.begin() : reader.getBlockContainsKey(beginKey, false), reader.end());
+ if (beginKey != null) {
+ inBlockAdvance(beginKey, false);
+ beginLocation.set(currentLocation);
+ }
+ if (endKey != null) {
+ seekTo(endKey, false);
+ endLocation.set(currentLocation);
+ seekTo(beginLocation);
+ }
+ }
+
+ /**
+ * Move the cursor to the first entry whose key is greater than or equal to the input key. Synonymous to seekTo(key, 0, key.length). The entry returned by
+ * the previous entry() call will be invalid.
+ *
+ * @param key
+ * The input key
+ * @return true if we find an equal key.
- * @throws IOException
+ */
+ public boolean seekTo(byte[] key) throws IOException {
+ return seekTo(key, 0, key.length);
+ }
+
+ /**
+ * Move the cursor to the first entry whose key is greater than or equal to the input key. The entry returned by the previous entry() call will be
+ * invalid.
+ *
+ * @param key
+ * The input key
+ * @param keyOffset
+ * offset in the key buffer.
+ * @param keyLen
+ * key buffer length.
+ * @return true if we find an equal key; false otherwise.
- * @throws IOException
+ */
+ public boolean seekTo(byte[] key, int keyOffset, int keyLen) throws IOException {
+ return seekTo(new ByteArray(key, keyOffset, keyLen), false);
+ }
+
+ private boolean seekTo(RawComparable key, boolean beyond) throws IOException {
+ Location l = reader.getBlockContainsKey(key, beyond);
+ if (l.compareTo(beginLocation) < 0) {
+ l = beginLocation;
+ } else if (l.compareTo(endLocation) >= 0) {
+ seekTo(endLocation);
+ return false;
+ }
+
+ // check if what we are seeking is in the later part of the current
+ // block.
+ if (atEnd() || (l.getBlockIndex() != currentLocation.getBlockIndex()) || (compareCursorKeyTo(key) >= 0)) {
+ // sorry, we must seek to a different location first.
+ seekTo(l);
+ }
+
+ return inBlockAdvance(key, beyond);
+ }
+
+ /**
+ * Move the cursor to the new location. The entry returned by the previous entry() call will be invalid.
+ *
+ * @param l
+ * new cursor location. It must fall between the begin and end location of the scanner.
+ * @throws IOException
+ */
+ private void seekTo(Location l) throws IOException {
+ if (l.compareTo(beginLocation) < 0) {
+ throw new IllegalArgumentException("Attempt to seek before the begin location.");
+ }
+
+ if (l.compareTo(endLocation) > 0) {
+ throw new IllegalArgumentException("Attempt to seek after the end location.");
+ }
+
+ if (l.compareTo(endLocation) == 0) {
+ parkCursorAtEnd();
+ return;
+ }
+
+ if (l.getBlockIndex() != currentLocation.getBlockIndex()) {
+ // going to a totally different block
+ initBlock(l.getBlockIndex());
+ } else {
+ if (valueChecked) {
+ // may temporarily go beyond the last record in the block (in which
+ // case the next if loop will always be true).
+ inBlockAdvance(1);
+ }
+ if (l.getRecordIndex() < currentLocation.getRecordIndex()) {
+ initBlock(l.getBlockIndex());
+ }
+ }
+
+ inBlockAdvance(l.getRecordIndex() - currentLocation.getRecordIndex());
+
+ return;
+ }
+
+ /**
+ * Rewind to the first entry in the scanner. The entry returned by the previous entry() call will be invalid.
- *
- * @throws IOException
+ */
+ public void rewind() throws IOException {
+ seekTo(beginLocation);
+ }
+
+ /**
+ * Seek to the end of the scanner. The entry returned by the previous entry() call will be invalid.
- *
- * @throws IOException
+ */
+ public void seekToEnd() throws IOException {
+ parkCursorAtEnd();
+ }
+
+ /**
+ * Move the cursor to the first entry whose key is greater than or equal to the input key. Synonymous to lowerBound(key, 0, key.length). The entry
+ * returned by the previous entry() call will be invalid.
+ *
+ * @param key
+ * The input key
- * @throws IOException
+ */
+ public void lowerBound(byte[] key) throws IOException {
+ lowerBound(key, 0, key.length);
+ }
+
+ /**
+ * Move the cursor to the first entry whose key is greater than or equal to the input key. The entry returned by the previous entry() call will be
+ * invalid.
+ *
+ * @param key
+ * The input key
+ * @param keyOffset
+ * offset in the key buffer.
+ * @param keyLen
+ * key buffer length.
- * @throws IOException
+ */
+ public void lowerBound(byte[] key, int keyOffset, int keyLen) throws IOException {
+ seekTo(new ByteArray(key, keyOffset, keyLen), false);
+ }
+
+ /**
+ * Move the cursor to the first entry whose key is strictly greater than the input key. Synonymous to upperBound(key, 0, key.length). The entry returned
+ * by the previous entry() call will be invalid.
+ *
+ * @param key
+ * The input key
- * @throws IOException
+ */
+ public void upperBound(byte[] key) throws IOException {
+ upperBound(key, 0, key.length);
+ }
+
+ /**
+ * Move the cursor to the first entry whose key is strictly greater than the input key. The entry returned by the previous entry() call will be invalid.
+ *
+ * @param key
+ * The input key
+ * @param keyOffset
+ * offset in the key buffer.
+ * @param keyLen
+ * key buffer length.
- * @throws IOException
+ */
+ public void upperBound(byte[] key, int keyOffset, int keyLen) throws IOException {
+ seekTo(new ByteArray(key, keyOffset, keyLen), true);
+ }
+
+ /**
+ * Move the cursor to the next key-value pair. The entry returned by the previous entry() call will be invalid.
+ *
+ * @return true if the cursor successfully moves. False when cursor is already at the end location and cannot be advanced.
- * @throws IOException
+ */
+ public boolean advance() throws IOException {
+ if (atEnd()) {
+ return false;
+ }
+
+ int curBid = currentLocation.getBlockIndex();
+ long curRid = currentLocation.getRecordIndex();
+ long entriesInBlock = reader.getBlockEntryCount(curBid);
+ if (curRid + 1 >= entriesInBlock) {
+ if (endLocation.compareTo(curBid + 1, 0) <= 0) {
+ // last entry in TFile.
+ parkCursorAtEnd();
+ } else {
+ // last entry in Block.
+ initBlock(curBid + 1);
+ }
+ } else {
+ inBlockAdvance(1);
+ }
+ return true;
+ }
+
+ /**
+ * Load a compressed block for reading. Expecting blockIndex is valid.
+ *
+ * @throws IOException
+ */
+ private void initBlock(int blockIndex) throws IOException {
+ klen = -1;
+ if (blkReader != null) {
+ try {
+ blkReader.close();
+ } finally {
+ blkReader = null;
+ }
+ }
+ blkReader = reader.getBlockReader(blockIndex);
+ currentLocation.set(blockIndex, 0);
+ }
+
+ private void parkCursorAtEnd() throws IOException {
+ klen = -1;
+ currentLocation.set(endLocation);
+ if (blkReader != null) {
+ try {
+ blkReader.close();
+ } finally {
+ blkReader = null;
+ }
+ }
+ }
+
+ /**
+ * Close the scanner. Release all resources. The behavior of using the scanner after calling close is not defined. The entry returned by the previous
+ * entry() call will be invalid.
+ */
+ public void close() throws IOException {
+ parkCursorAtEnd();
+ }
+
+ /**
+ * Is cursor at the end location?
+ *
+ * @return true if the cursor is at the end location.
+ */
+ public boolean atEnd() {
+ return (currentLocation.compareTo(endLocation) >= 0);
+ }
+
+ /**
+ * check whether we have already successfully obtained the key. It also initializes the valueInputStream.
+ */
+ void checkKey() throws IOException {
+ if (klen >= 0)
+ return;
+ if (atEnd()) {
+ throw new EOFException("No key-value to read");
+ }
+ klen = -1;
+ vlen = -1;
+ valueChecked = false;
+
+ klen = Utils.readVInt(blkReader);
+ blkReader.readFully(keyBuffer, 0, klen);
+ valueBufferInputStream.reset(blkReader);
+ if (valueBufferInputStream.isLastChunk()) {
+ vlen = valueBufferInputStream.getRemain();
+ }
+ }
+
+ /**
+ * Get an entry to access the key and value.
+ *
+ * @return The Entry object to access the key and value.
- * @throws IOException
+ */
+ public Entry entry() throws IOException {
+ checkKey();
+ return new Entry();
+ }
+
+ /**
+ * Internal API. Comparing the key at cursor to user-specified key.
+ *
+ * @param other
+ * user-specified key.
+ * @return negative if key at cursor is smaller than user key; 0 if equal; and positive if key at cursor greater than user key.
+ * @throws IOException
+ */
+ int compareCursorKeyTo(RawComparable other) throws IOException {
+ checkKey();
+ return reader.compareKeys(keyBuffer, 0, klen, other.buffer(), other.offset(), other.size());
+ }
+
+ /**
+ * Entry to a <Key, Value> pair.
+ */
+ public class Entry implements Comparable<RawComparable> {
+ /**
+ * Get the length of the key.
+ *
+ * @return the length of the key.
+ */
+ public int getKeyLength() {
+ return klen;
+ }
+
+ byte[] getKeyBuffer() {
+ return keyBuffer;
+ }
+
+ /**
+ * Copy the key and value in one shot into BytesWritables. This is equivalent to getKey(key); getValue(value);
+ *
+ * @param key
+ * BytesWritable to hold key.
+ * @param value
+ * BytesWritable to hold value
- * @throws IOException
+ */
+ public void get(BytesWritable key, BytesWritable value) throws IOException {
+ getKey(key);
+ getValue(value);
+ }
+
+ /**
+ * Copy the key into BytesWritable. The input BytesWritable will be automatically resized to the actual key size.
+ *
+ * @param key
+ * BytesWritable to hold the key.
- * @throws IOException
+ */
+ public int getKey(BytesWritable key) throws IOException {
+ key.setSize(getKeyLength());
+ getKey(key.getBytes());
+ return key.getLength();
+ }
+
+ /**
+ * Copy the value into BytesWritable. The input BytesWritable will be automatically resized to the actual value size. The implementation directly uses
+ * the buffer inside BytesWritable for storing the value. The call does not require the value length to be known.
- *
- * @param value
- * @throws IOException
+ */
+ public long getValue(BytesWritable value) throws IOException {
+ DataInputStream dis = getValueStream();
+ int size = 0;
+ try {
+ int remain;
+ while ((remain = valueBufferInputStream.getRemain()) > 0) {
+ value.setSize(size + remain);
+ dis.readFully(value.getBytes(), size, remain);
+ size += remain;
+ }
+ return value.getLength();
+ } finally {
+ dis.close();
+ }
+ }
+
+ /**
+ * Writing the key to the output stream. This method avoids copying key buffer from Scanner into user buffer, then writing to the output stream.
+ *
+ * @param out
+ * The output stream
+ * @return the length of the key.
- * @throws IOException
+ */
+ public int writeKey(OutputStream out) throws IOException {
+ out.write(keyBuffer, 0, klen);
+ return klen;
+ }
+
+ /**
+ * Writing the value to the output stream. This method avoids copying value data from Scanner into user buffer, then writing to the output stream. It
+ * does not require the value length to be known.
+ *
+ * @param out
+ * The output stream
+ * @return the length of the value
- * @throws IOException
+ */
+ public long writeValue(OutputStream out) throws IOException {
+ DataInputStream dis = getValueStream();
+ long size = 0;
+ try {
+ int chunkSize;
+ while ((chunkSize = valueBufferInputStream.getRemain()) > 0) {
+ chunkSize = Math.min(chunkSize, MAX_VAL_TRANSFER_BUF_SIZE);
+ valTransferBuffer.setSize(chunkSize);
+ dis.readFully(valTransferBuffer.getBytes(), 0, chunkSize);
+ out.write(valTransferBuffer.getBytes(), 0, chunkSize);
+ size += chunkSize;
+ }
+ return size;
+ } finally {
+ dis.close();
+ }
+ }
+
+ /**
+ * Copy the key into user supplied buffer.
+ *
+ * @param buf
+ * The buffer supplied by user. The length of the buffer must not be shorter than the key length.
+ * @return The length of the key.
- *
- * @throws IOException
+ */
+ public int getKey(byte[] buf) throws IOException {
+ return getKey(buf, 0);
+ }
+
+ /**
+ * Copy the key into user supplied buffer.
+ *
+ * @param buf
+ * The buffer supplied by user.
+ * @param offset
+ * The starting offset of the user buffer where we should copy the key into. Requiring the key-length + offset no greater than the buffer
+ * length.
+ * @return The length of the key.
- * @throws IOException
+ */
+ public int getKey(byte[] buf, int offset) throws IOException {
+ if ((offset | (buf.length - offset - klen)) < 0) {
+ throw new IndexOutOfBoundsException("Bufer not enough to store the key");
+ }
+ System.arraycopy(keyBuffer, 0, buf, offset, klen);
+ return klen;
+ }
+
+ /**
+ * Streaming access to the key. Useful for desrializing the key into user objects.
+ *
+ * @return The input stream.
+ */
+ public DataInputStream getKeyStream() {
+ keyDataInputStream.reset(keyBuffer, klen);
+ return keyDataInputStream;
+ }
+
+ /**
+ * Get the length of the value. isValueLengthKnown() must be tested true.
+ *
+ * @return the length of the value.
+ */
+ public int getValueLength() {
+ if (vlen >= 0) {
+ return vlen;
+ }
+
+ throw new RuntimeException("Value length unknown.");
+ }
+
+ /**
+ * Copy value into user-supplied buffer. User supplied buffer must be large enough to hold the whole value. The value part of the key-value pair pointed
+ * by the current cursor is not cached and can only be examined once. Calling any of the following functions more than once without moving the cursor
+ * will result in exception: {@link #getValue(byte[])}, {@link #getValue(byte[], int)}, {@link #getValueStream}.
+ *
+ * @return the length of the value. Does not require isValueLengthKnown() to be true.
- * @throws IOException
+ *
+ */
+ public int getValue(byte[] buf) throws IOException {
+ return getValue(buf, 0);
+ }
+
+ /**
+ * Copy value into user-supplied buffer. User supplied buffer must be large enough to hold the whole value (starting from the offset). The value part of
+ * the key-value pair pointed by the current cursor is not cached and can only be examined once. Calling any of the following functions more than once
+ * without moving the cursor will result in exception: {@link #getValue(byte[])}, {@link #getValue(byte[], int)}, {@link #getValueStream}.
+ *
+ * @return the length of the value. Does not require isValueLengthKnown() to be true.
- * @throws IOException
+ */
+ public int getValue(byte[] buf, int offset) throws IOException {
+ DataInputStream dis = getValueStream();
+ try {
+ if (isValueLengthKnown()) {
+ if ((offset | (buf.length - offset - vlen)) < 0) {
+ throw new IndexOutOfBoundsException("Buffer too small to hold value");
+ }
+ dis.readFully(buf, offset, vlen);
+ return vlen;
+ }
+
+ int nextOffset = offset;
+ while (nextOffset < buf.length) {
+ int n = dis.read(buf, nextOffset, buf.length - nextOffset);
+ if (n < 0) {
+ break;
+ }
+ nextOffset += n;
+ }
+ if (dis.read() >= 0) {
+ // attempt to read one more byte to determine whether we reached
+ // the
+ // end or not.
+ throw new IndexOutOfBoundsException("Buffer too small to hold value");
+ }
+ return nextOffset - offset;
+ } finally {
+ dis.close();
+ }
+ }
+
+ /**
+ * Stream access to value. The value part of the key-value pair pointed by the current cursor is not cached and can only be examined once. Calling any
+ * of the following functions more than once without moving the cursor will result in exception: {@link #getValue(byte[])},
+ * {@link #getValue(byte[], int)}, {@link #getValueStream}.
+ *
+ * @return The input stream for reading the value.
- * @throws IOException
+ */
+ public DataInputStream getValueStream() throws IOException {
+ if (valueChecked == true) {
+ throw new IllegalStateException("Attempt to examine value multiple times.");
+ }
+ valueChecked = true;
+ return valueDataInputStream;
+ }
+
+ /**
+ * Check whether it is safe to call getValueLength().
+ *
+ * @return true if value length is known before hand. Values less than the chunk size will always have their lengths known before hand. Values that are
+ * written out as a whole (with advertised length up-front) will always have their lengths known in read.
+ */
+ public boolean isValueLengthKnown() {
+ return (vlen >= 0);
+ }
+
+ /**
+ * Compare the entry key to another key. Synonymous to compareTo(key, 0, key.length).
+ *
+ * @param buf
+ * The key buffer.
+ * @return comparison result between the entry key with the input key.
+ */
+ public int compareTo(byte[] buf) {
+ return compareTo(buf, 0, buf.length);
+ }
+
+ /**
+ * Compare the entry key to another key. Synonymous to compareTo(new ByteArray(buf, offset, length)
+ *
+ * @param buf
+ * The key buffer
+ * @param offset
+ * offset into the key buffer.
+ * @param length
+ * the length of the key.
+ * @return comparison result between the entry key with the input key.
+ */
+ public int compareTo(byte[] buf, int offset, int length) {
+ return compareTo(new ByteArray(buf, offset, length));
+ }
+
+ /**
+ * Compare an entry with a RawComparable object. This is useful when Entries are stored in a collection, and we want to compare a user supplied key.
+ */
+ @Override
+ public int compareTo(RawComparable key) {
+ return reader.compareKeys(keyBuffer, 0, getKeyLength(), key.buffer(), key.offset(), key.size());
+ }
+
+ /**
+ * Compare whether this and other points to the same key value.
+ */
+ @Override
+ public boolean equals(Object other) {
+ if (this == other)
+ return true;
+ if (!(other instanceof Entry))
+ return false;
+ return ((Entry) other).compareTo(keyBuffer, 0, getKeyLength()) == 0;
+ }
+
+ @Override
+ public int hashCode() {
+ return WritableComparator.hashBytes(keyBuffer, getKeyLength());
+ }
+ }
+
+ /**
+ * Advance cursor by n positions within the block.
+ *
+ * @param n
+ * Number of key-value pairs to skip in block.
+ * @throws IOException
+ */
+ private void inBlockAdvance(long n) throws IOException {
+ for (long i = 0; i < n; ++i) {
+ checkKey();
+ if (!valueBufferInputStream.isClosed()) {
+ valueBufferInputStream.close();
+ }
+ klen = -1;
+ currentLocation.incRecordIndex();
+ }
+ }
+
+ /**
+ * Advance cursor in block until we find a key that is greater than or equal to the input key.
+ *
+ * @param key
+ * Key to compare.
+ * @param greater
+ * advance until we find a key greater than the input key.
+ * @return true if we find a equal key.
+ * @throws IOException
+ */
+ private boolean inBlockAdvance(RawComparable key, boolean greater) throws IOException {
+ int curBid = currentLocation.getBlockIndex();
+ long entryInBlock = reader.getBlockEntryCount(curBid);
+ if (curBid == endLocation.getBlockIndex()) {
+ entryInBlock = endLocation.getRecordIndex();
+ }
+
+ while (currentLocation.getRecordIndex() < entryInBlock) {
+ int cmp = compareCursorKeyTo(key);
+ if (cmp > 0)
+ return false;
+ if (cmp == 0 && !greater)
+ return true;
+ if (!valueBufferInputStream.isClosed()) {
+ valueBufferInputStream.close();
+ }
+ klen = -1;
+ currentLocation.incRecordIndex();
+ }
+
+ throw new RuntimeException("Cannot find matching key in block.");
+ }
+ }
+
+ long getBlockEntryCount(int curBid) {
+ return tfileIndex.getEntry(curBid).entries();
+ }
+
+ BlockReader getBlockReader(int blockIndex) throws IOException {
+ return readerBCF.getDataBlock(blockIndex);
+ }
+ }
+
+ /**
+ * Data structure representing "TFile.meta" meta block.
+ */
+ static final class TFileMeta {
+ final static String BLOCK_NAME = "TFile.meta";
+ final Version version;
+ private long recordCount;
+ private final String strComparator;
+ private final BytesComparator comparator;
+
+ // ctor for writes
+ public TFileMeta(String comparator) {
+ // set fileVersion to API version when we create it.
+ version = TFile.API_VERSION;
+ recordCount = 0;
+ strComparator = (comparator == null) ? "" : comparator;
+ this.comparator = makeComparator(strComparator);
+ }
+
+ // ctor for reads
+ public TFileMeta(DataInput in) throws IOException {
+ version = new Version(in);
+ if (!version.compatibleWith(TFile.API_VERSION)) {
+ throw new RuntimeException("Incompatible TFile fileVersion.");
+ }
+ recordCount = Utils.readVLong(in);
+ strComparator = Utils.readString(in);
+ comparator = makeComparator(strComparator);
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ static BytesComparator makeComparator(String comparator) {
+ if (comparator.length() == 0) {
+ // unsorted keys
+ return null;
+ }
+ if (comparator.equals(COMPARATOR_MEMCMP)) {
+ // default comparator
+ return new BytesComparator(new MemcmpRawComparator());
+ } else if (comparator.startsWith(COMPARATOR_JCLASS)) {
+ String compClassName = comparator.substring(COMPARATOR_JCLASS.length()).trim();
+ try {
+ Class compClass = Class.forName(compClassName);
+ // use its default ctor to create an instance
+ return new BytesComparator((RawComparator<Object>) compClass.newInstance());
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Failed to instantiate comparator: " + comparator + "(" + e.toString() + ")");
+ }
+ } else {
+ throw new IllegalArgumentException("Unsupported comparator: " + comparator);
+ }
+ }
+
+ public void write(DataOutput out) throws IOException {
+ TFile.API_VERSION.write(out);
+ Utils.writeVLong(out, recordCount);
+ Utils.writeString(out, strComparator);
+ }
+
+ public long getRecordCount() {
+ return recordCount;
+ }
+
+ public void incRecordCount() {
+ ++recordCount;
+ }
+
+ public boolean isSorted() {
+ return !strComparator.equals("");
+ }
+
+ public String getComparatorString() {
+ return strComparator;
+ }
+
+ public BytesComparator getComparator() {
+ return comparator;
+ }
+
+ public Version getVersion() {
+ return version;
+ }
+ } // END: class MetaTFileMeta
+
+ /**
+ * Data structure representing "TFile.index" meta block.
+ */
+ static class TFileIndex {
+ final static String BLOCK_NAME = "TFile.index";
+ private ByteArray firstKey;
+ private final ArrayList<TFileIndexEntry> index;
+ private final BytesComparator comparator;
+
+ /**
+ * For reading from file.
- *
- * @throws IOException
+ */
+ public TFileIndex(int entryCount, DataInput in, BytesComparator comparator) throws IOException {
+ index = new ArrayList<TFileIndexEntry>(entryCount);
+ int size = Utils.readVInt(in); // size for the first key entry.
+ if (size > 0) {
+ byte[] buffer = new byte[size];
+ in.readFully(buffer);
+ DataInputStream firstKeyInputStream = new DataInputStream(new ByteArrayInputStream(buffer, 0, size));
+
+ int firstKeyLength = Utils.readVInt(firstKeyInputStream);
+ firstKey = new ByteArray(new byte[firstKeyLength]);
+ firstKeyInputStream.readFully(firstKey.buffer());
+
+ for (int i = 0; i < entryCount; i++) {
+ size = Utils.readVInt(in);
+ if (buffer.length < size) {
+ buffer = new byte[size];
+ }
+ in.readFully(buffer, 0, size);
+ TFileIndexEntry idx = new TFileIndexEntry(new DataInputStream(new ByteArrayInputStream(buffer, 0, size)));
+ index.add(idx);
+ }
+ } else {
+ if (entryCount != 0) {
+ throw new RuntimeException("Internal error");
+ }
+ }
+ this.comparator = comparator;
+ }
+
+ /**
+ * @param key
+ * input key.
+ * @return the ID of the first block that contains key >= input key. Or -1 if no such block exists.
+ */
+ public int lowerBound(RawComparable key) {
+ if (comparator == null) {
+ throw new RuntimeException("Cannot search in unsorted TFile");
+ }
+
+ if (firstKey == null) {
+ return -1; // not found
+ }
+
+ int ret = Utils.lowerBound(index, key, comparator);
+ if (ret == index.size()) {
+ return -1;
+ }
+ return ret;
+ }
+
+ public int upperBound(RawComparable key) {
+ if (comparator == null) {
+ throw new RuntimeException("Cannot search in unsorted TFile");
+ }
+
+ if (firstKey == null) {
+ return -1; // not found
+ }
+
+ int ret = Utils.upperBound(index, key, comparator);
+ if (ret == index.size()) {
+ return -1;
+ }
+ return ret;
+ }
+
+ /**
+ * For writing to file.
+ */
+ public TFileIndex(BytesComparator comparator) {
+ index = new ArrayList<TFileIndexEntry>();
+ this.comparator = comparator;
+ }
+
+ public RawComparable getFirstKey() {
+ return firstKey;
+ }
+
+ public void setFirstKey(byte[] key, int offset, int length) {
+ firstKey = new ByteArray(new byte[length]);
+ System.arraycopy(key, offset, firstKey.buffer(), 0, length);
+ }
+
+ public RawComparable getLastKey() {
+ if (index.size() == 0) {
+ return null;
+ }
+ return new ByteArray(index.get(index.size() - 1).buffer());
+ }
+
+ public void addEntry(TFileIndexEntry keyEntry) {
+ index.add(keyEntry);
+ }
+
+ public TFileIndexEntry getEntry(int bid) {
+ return index.get(bid);
+ }
+
+ public void write(DataOutput out) throws IOException {
+ if (firstKey == null) {
+ Utils.writeVInt(out, 0);
+ return;
+ }
+
+ DataOutputBuffer dob = new DataOutputBuffer();
+ Utils.writeVInt(dob, firstKey.size());
+ dob.write(firstKey.buffer());
+ Utils.writeVInt(out, dob.size());
+ out.write(dob.getData(), 0, dob.getLength());
+
+ for (TFileIndexEntry entry : index) {
+ dob.reset();
+ entry.write(dob);
+ Utils.writeVInt(out, dob.getLength());
+ out.write(dob.getData(), 0, dob.getLength());
+ }
+ }
+ }
+
+ /**
+ * TFile Data Index entry. We should try to make the memory footprint of each index entry as small as possible.
+ */
+ static final class TFileIndexEntry implements RawComparable {
+ final byte[] key;
+ // count of <key, value> entries in the block.
+ final long kvEntries;
+
+ public TFileIndexEntry(DataInput in) throws IOException {
+ int len = Utils.readVInt(in);
+ key = new byte[len];
+ in.readFully(key, 0, len);
+ kvEntries = Utils.readVLong(in);
+ }
+
+ // default entry, without any padding
+ public TFileIndexEntry(byte[] newkey, int offset, int len, long entries) {
+ key = new byte[len];
+ System.arraycopy(newkey, offset, key, 0, len);
+ this.kvEntries = entries;
+ }
+
+ @Override
+ public byte[] buffer() {
+ return key;
+ }
+
+ @Override
+ public int offset() {
+ return 0;
+ }
+
+ @Override
+ public int size() {
+ return key.length;
+ }
+
+ long entries() {
+ return kvEntries;
+ }
+
+ public void write(DataOutput out) throws IOException {
+ Utils.writeVInt(out, key.length);
+ out.write(key, 0, key.length);
+ Utils.writeVLong(out, kvEntries);
+ }
+ }
+
+ /**
+ * Dumping the TFile information.
+ *
+ * @param args
+ * A list of TFile paths.
+ */
+ public static void main(String[] args) {
+ System.out.printf("TFile Dumper (TFile %s, BCFile %s)%n", TFile.API_VERSION.toString(), BCFile.API_VERSION.toString());
+ if (args.length == 0) {
+ System.out.println("Usage: java ... org.apache.hadoop.io.file.tfile.TFile tfile-path [tfile-path ...]");
+ System.exit(0);
+ }
+ Configuration conf = new Configuration();
+
+ for (String file : args) {
+ System.out.println("===" + file + "===");
+ try {
+ TFileDumper.dumpInfo(file, System.out, conf);
+ } catch (IOException e) {
+ e.printStackTrace(System.err);
+ }
+ }
+ }
+}
[57/64] [abbrv] git commit: Merge branch '1.5.2-SNAPSHOT' into
1.6.0-SNAPSHOT
Posted by ct...@apache.org.
Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Conflicts:
core/src/main/java/org/apache/accumulo/core/Constants.java
core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
core/src/main/java/org/apache/accumulo/core/data/ColumnUpdate.java
core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Chunk.java
core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFile.java
core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFileDumper.java
core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModule.java
core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
core/src/test/java/org/apache/accumulo/core/util/shell/command/FormatterCommandTest.java
examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
server/base/src/main/java/org/apache/accumulo/server/util/TServerUtils.java
server/src/main/java/org/apache/accumulo/server/metanalysis/LogFileOutputFormat.java
server/src/main/java/org/apache/accumulo/server/metanalysis/PrintEvents.java
server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java
test/src/main/java/org/apache/accumulo/test/GetMasterStats.java
test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/716ea0ee
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/716ea0ee
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/716ea0ee
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 716ea0ee8b26bf504d0cf9e90fc1d3d8579bc50a
Parents: 3934ea6 9261338
Author: Christopher Tubbs <ct...@apache.org>
Authored: Wed Apr 9 13:36:22 2014 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Wed Apr 9 13:36:22 2014 -0400
----------------------------------------------------------------------
.../core/client/ClientSideIteratorScanner.java | 2 -
.../accumulo/core/client/ConditionalWriter.java | 2 -
.../core/client/ConditionalWriterConfig.java | 2 -
.../apache/accumulo/core/client/Connector.java | 2 -
.../accumulo/core/client/IteratorSetting.java | 4 -
.../accumulo/core/client/RowIterator.java | 4 -
.../accumulo/core/client/ScannerBase.java | 1 -
.../core/client/admin/ActiveCompaction.java | 1 -
.../core/client/admin/InstanceOperations.java | 12 --
.../core/client/admin/TableOperations.java | 30 ----
.../core/client/admin/TableOperationsImpl.java | 4 -
.../core/client/impl/ThriftTransportPool.java | 16 +-
.../core/client/mapred/AbstractInputFormat.java | 2 -
.../client/mapred/AccumuloOutputFormat.java | 2 -
.../client/mapreduce/AbstractInputFormat.java | 3 +-
.../client/mapreduce/AccumuloOutputFormat.java | 2 -
.../core/client/mapreduce/InputTableConfig.java | 3 -
.../mapreduce/lib/util/ConfiguratorBase.java | 2 -
.../core/client/mock/MockBatchDeleter.java | 4 -
.../apache/accumulo/core/data/Condition.java | 7 -
.../java/org/apache/accumulo/core/data/Key.java | 7 +-
.../org/apache/accumulo/core/data/Range.java | 17 +-
.../accumulo/core/file/rfile/BlockIndex.java | 5 -
.../accumulo/core/file/rfile/bcfile/BCFile.java | 15 +-
.../core/file/rfile/bcfile/ByteArray.java | 2 -
.../accumulo/core/file/rfile/bcfile/Utils.java | 11 --
.../core/iterators/TypedValueCombiner.java | 6 -
.../core/iterators/ValueFormatException.java | 6 -
.../core/iterators/system/MapFileIterator.java | 8 +-
.../core/iterators/user/GrepIterator.java | 3 -
.../iterators/user/IntersectingIterator.java | 10 --
.../accumulo/core/iterators/user/RowFilter.java | 1 -
.../iterators/user/TransformingIterator.java | 164 +++++++++----------
.../core/iterators/user/VersioningIterator.java | 3 -
.../accumulo/core/security/SecurityUtil.java | 1 -
.../security/crypto/CryptoModuleFactory.java | 1 -
.../security/crypto/CryptoModuleParameters.java | 6 -
.../core/client/impl/ScannerOptionsTest.java | 2 -
.../client/lexicoder/ReverseLexicoderTest.java | 2 -
.../client/mapred/AccumuloInputFormatTest.java | 4 -
.../mapreduce/AccumuloInputFormatTest.java | 4 -
.../core/client/mock/MockNamespacesTest.java | 8 -
.../core/security/VisibilityConstraintTest.java | 3 -
.../simple/client/RandomBatchScanner.java | 5 -
.../simple/client/RandomBatchWriter.java | 4 -
.../simple/client/SequentialBatchWriter.java | 5 -
.../simple/client/TraceDumpExample.java | 9 +-
.../examples/simple/dirlist/QueryUtil.java | 3 -
.../examples/simple/mapreduce/NGramIngest.java | 3 -
.../examples/simple/mapreduce/TableToFile.java | 1 -
.../accumulo/examples/simple/shard/Query.java | 3 -
.../minicluster/MiniAccumuloRunner.java | 2 -
.../accumulo/proxy/TestProxyReadWrite.java | 10 --
.../accumulo/server/conf/ConfigSanityCheck.java | 3 -
.../server/master/balancer/TabletBalancer.java | 2 -
.../server/master/state/TabletStateStore.java | 7 -
.../server/metrics/AbstractMetricsImpl.java | 4 -
.../server/security/handler/ZKAuthorizor.java | 4 -
.../server/security/handler/ZKPermHandler.java | 4 -
.../accumulo/server/util/LoginProperties.java | 3 -
.../accumulo/server/util/RestoreZookeeper.java | 4 -
.../accumulo/server/util/TableDiskUsage.java | 3 -
.../accumulo/server/util/TabletServerLocks.java | 3 -
.../org/apache/accumulo/tserver/MemValue.java | 4 +-
.../apache/accumulo/tserver/TabletServer.java | 16 +-
.../tserver/compaction/CompactionStrategy.java | 4 -
.../accumulo/tserver/logger/LogReader.java | 1 -
.../start/classloader/AccumuloClassLoader.java | 3 -
.../start/classloader/vfs/ContextManager.java | 2 -
.../vfs/PostDelegatingVFSClassLoader.java | 7 +-
.../vfs/providers/HdfsFileSystem.java | 5 -
.../accumulo/test/NativeMapPerformanceTest.java | 3 -
.../accumulo/test/NativeMapStressTest.java | 3 -
.../test/continuous/ContinuousMoru.java | 1 -
.../test/continuous/ContinuousVerify.java | 1 -
.../test/functional/CacheTestClean.java | 3 -
.../accumulo/test/randomwalk/Framework.java | 3 -
.../apache/accumulo/test/randomwalk/Node.java | 1 -
.../randomwalk/concurrent/CheckBalance.java | 5 +-
79 files changed, 124 insertions(+), 414 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/ConditionalWriter.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/ConditionalWriter.java
index 5fdccf0,0000000..95f73bb
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/ConditionalWriter.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ConditionalWriter.java
@@@ -1,141 -1,0 +1,139 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.core.client;
+
+import java.util.Iterator;
+
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.data.ConditionalMutation;
+
+/**
+ * ConditionalWriter provides the ability to do efficient, atomic read-modify-write operations on rows. These operations are performed on the tablet server
+ * while a row lock is held.
+ *
+ * @since 1.6.0
+ */
+public interface ConditionalWriter {
+ class Result {
+
+ private Status status;
+ private ConditionalMutation mutation;
+ private String server;
+ private Exception exception;
+
+ public Result(Status s, ConditionalMutation m, String server) {
+ this.status = s;
+ this.mutation = m;
+ this.server = server;
+ }
+
+ public Result(Exception e, ConditionalMutation cm, String server) {
+ this.exception = e;
+ this.mutation = cm;
+ this.server = server;
+ }
+
+ /**
+ * If this method throws an exception, then its possible the mutation is still being actively processed. Therefore if code chooses to continue after seeing
+ * an exception it should take this into consideration.
+ *
+ * @return status of a conditional mutation
- * @throws AccumuloException
- * @throws AccumuloSecurityException
+ */
+
+ public Status getStatus() throws AccumuloException, AccumuloSecurityException {
+ if (status == null) {
+ if (exception instanceof AccumuloException)
+ throw new AccumuloException(exception);
+ if (exception instanceof AccumuloSecurityException) {
+ AccumuloSecurityException ase = (AccumuloSecurityException) exception;
+ throw new AccumuloSecurityException(ase.getUser(), SecurityErrorCode.valueOf(ase.getSecurityErrorCode().name()), ase.getTableInfo(), ase);
+ }
+ else
+ throw new AccumuloException(exception);
+ }
+
+ return status;
+ }
+
+ /**
+ *
+ * @return A copy of the mutation previously submitted by a user. The mutation will reference the same data, but the object may be different.
+ */
+ public ConditionalMutation getMutation() {
+ return mutation;
+ }
+
+ /**
+ *
+ * @return The server this mutation was sent to. Returns null if was not sent to a server.
+ */
+ public String getTabletServer() {
+ return server;
+ }
+ }
+
+ public static enum Status {
+ /**
+ * conditions were met and mutation was written
+ */
+ ACCEPTED,
+ /**
+ * conditions were not met and mutation was not written
+ */
+ REJECTED,
+ /**
+ * mutation violated a constraint and was not written
+ */
+ VIOLATED,
+ /**
+ * error occurred after mutation was sent to server, its unknown if the mutation was written. Although the status of the mutation is unknown, Accumulo
+ * guarantees the mutation will not be written at a later point in time.
+ */
+ UNKNOWN,
+ /**
+ * A condition contained a column visibility that could never be seen
+ */
+ INVISIBLE_VISIBILITY,
+
+ }
+
+ /**
+ * This method returns one result for each mutation passed to it. This method is thread safe. Multiple threads can safely use a single conditional writer.
+ * Sharing a conditional writer between multiple threads may result in batching of request to tablet servers.
+ *
+ * @param mutations
+ * @return Result for each mutation submitted. The mutations may still be processing in the background when this method returns, if so the iterator will
+ * block.
+ */
+ Iterator<Result> write(Iterator<ConditionalMutation> mutations);
+
+ /**
+ * This method has the same thread safety guarantees as @link {@link #write(Iterator)}
+ *
+ *
+ * @param mutation
+ * @return Result for the submitted mutation
+ */
+
+ Result write(ConditionalMutation mutation);
+
+ /**
+ * release any resources (like threads pools) used by conditional writer
+ */
+ void close();
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/ConditionalWriterConfig.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/ConditionalWriterConfig.java
index f2a91ea,0000000..a220e62
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/ConditionalWriterConfig.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ConditionalWriterConfig.java
@@@ -1,118 -1,0 +1,116 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.ArgumentChecker;
+
+/**
+ *
+ * @since 1.6.0
+ */
+public class ConditionalWriterConfig {
+
+ private static final Long DEFAULT_TIMEOUT = Long.MAX_VALUE;
+ private Long timeout = null;
+
+ private static final Integer DEFAULT_MAX_WRITE_THREADS = 3;
+ private Integer maxWriteThreads = null;
+
+ private Authorizations auths = Authorizations.EMPTY;
+
+ /**
+ * A set of authorization labels that will be checked against the column visibility of each key in order to filter data. The authorizations passed in must be
+ * a subset of the accumulo user's set of authorizations. If the accumulo user has authorizations (A1, A2) and authorizations (A2, A3) are passed, then an
+ * exception will be thrown.
+ *
+ * <p>
+ * Any condition that is not visible with this set of authorizations will fail.
- *
- * @param auths
+ */
+ public ConditionalWriterConfig setAuthorizations(Authorizations auths) {
+ ArgumentChecker.notNull(auths);
+ this.auths = auths;
+ return this;
+ }
+
+ /**
+ * Sets the maximum amount of time an unresponsive server will be re-tried. When this timeout is exceeded, the {@link ConditionalWriter} should return the
+ * mutation with an exception.<br />
+ * For no timeout, set to zero, or {@link Long#MAX_VALUE} with {@link TimeUnit#MILLISECONDS}.
+ *
+ * <p>
+ * {@link TimeUnit#MICROSECONDS} or {@link TimeUnit#NANOSECONDS} will be truncated to the nearest {@link TimeUnit#MILLISECONDS}.<br />
+ * If this truncation would result in making the value zero when it was specified as non-zero, then a minimum value of one {@link TimeUnit#MILLISECONDS} will
+ * be used.
+ *
+ * <p>
+ * <b>Default:</b> {@link Long#MAX_VALUE} (no timeout)
+ *
+ * @param timeout
+ * the timeout, in the unit specified by the value of {@code timeUnit}
+ * @param timeUnit
+ * determines how {@code timeout} will be interpreted
+ * @throws IllegalArgumentException
+ * if {@code timeout} is less than 0
+ * @return {@code this} to allow chaining of set methods
+ */
+ public ConditionalWriterConfig setTimeout(long timeout, TimeUnit timeUnit) {
+ if (timeout < 0)
+ throw new IllegalArgumentException("Negative timeout not allowed " + timeout);
+
+ if (timeout == 0)
+ this.timeout = Long.MAX_VALUE;
+ else
+ // make small, positive values that truncate to 0 when converted use the minimum millis instead
+ this.timeout = Math.max(1, timeUnit.toMillis(timeout));
+ return this;
+ }
+
+ /**
+ * Sets the maximum number of threads to use for writing data to the tablet servers.
+ *
+ * <p>
+ * <b>Default:</b> 3
+ *
+ * @param maxWriteThreads
+ * the maximum threads to use
+ * @throws IllegalArgumentException
+ * if {@code maxWriteThreads} is non-positive
+ * @return {@code this} to allow chaining of set methods
+ */
+ public ConditionalWriterConfig setMaxWriteThreads(int maxWriteThreads) {
+ if (maxWriteThreads <= 0)
+ throw new IllegalArgumentException("Max threads must be positive " + maxWriteThreads);
+
+ this.maxWriteThreads = maxWriteThreads;
+ return this;
+ }
+
+ public Authorizations getAuthorizations() {
+ return auths;
+ }
+
+ public long getTimeout(TimeUnit timeUnit) {
+ return timeUnit.convert(timeout != null ? timeout : DEFAULT_TIMEOUT, TimeUnit.MILLISECONDS);
+ }
+
+ public int getMaxWriteThreads() {
+ return maxWriteThreads != null ? maxWriteThreads : DEFAULT_MAX_WRITE_THREADS;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/Connector.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/Connector.java
index 92a1184,3189d44..4a2acff
--- a/core/src/main/java/org/apache/accumulo/core/client/Connector.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/Connector.java
@@@ -88,13 -87,12 +88,12 @@@ public abstract class Connector
* @param config
* configuration used to create batch writer
* @return BatchDeleter object for configuring and deleting
- * @throws TableNotFoundException
* @since 1.5.0
*/
-
+
public abstract BatchDeleter createBatchDeleter(String tableName, Authorizations authorizations, int numQueryThreads, BatchWriterConfig config)
throws TableNotFoundException;
-
+
/**
* Factory method to create a BatchWriter connected to Accumulo.
*
@@@ -123,12 -121,11 +122,11 @@@
* @param config
* configuration used to create batch writer
* @return BatchWriter object for configuring and writing data to
- * @throws TableNotFoundException
* @since 1.5.0
*/
-
+
public abstract BatchWriter createBatchWriter(String tableName, BatchWriterConfig config) throws TableNotFoundException;
-
+
/**
* Factory method to create a Multi-Table BatchWriter connected to Accumulo. Multi-table batch writers can queue data for multiple tables, which is good for
* ingesting data into multiple tables from the same source
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
index 7a98df2,e58a1be..e69f3dd
--- a/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
@@@ -82,11 -82,9 +82,9 @@@ public class IteratorSetting implement
public String getName() {
return name;
}
-
+
/**
* Set the iterator's name. Must be a simple alphanumeric identifier.
- *
- * @param name
*/
public void setName(String name) {
ArgumentChecker.notNull(name);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
index afa539a,29ff2a6..7b58bd4
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
@@@ -58,21 -58,17 +58,17 @@@ public interface InstanceOperations
*
* @return A map of system properties set in zookeeper. If a property is not set in zookeeper, then it will return the value set in accumulo-site.xml on some
* server. If nothing is set in an accumulo-site.xml file it will return the default value for each property.
- * @throws AccumuloException
- * @throws AccumuloSecurityException
*/
- public Map<String,String> getSystemConfiguration() throws AccumuloException, AccumuloSecurityException;
+ Map<String,String> getSystemConfiguration() throws AccumuloException, AccumuloSecurityException;
/**
*
* @return A map of system properties set in accumulo-site.xml on some server. If nothing is set in an accumulo-site.xml file it will return the default value
* for each property.
- * @throws AccumuloException
- * @throws AccumuloSecurityException
*/
- public Map<String,String> getSiteConfiguration() throws AccumuloException, AccumuloSecurityException;
+ Map<String,String> getSiteConfiguration() throws AccumuloException, AccumuloSecurityException;
/**
* List the currently active tablet servers participating in the accumulo instance
@@@ -88,11 -84,9 +84,9 @@@
* @param tserver
* The tablet server address should be of the form <ip address>:<port>
* @return A list of active scans on tablet server.
- * @throws AccumuloException
- * @throws AccumuloSecurityException
*/
- public List<ActiveScan> getActiveScans(String tserver) throws AccumuloException, AccumuloSecurityException;
+ List<ActiveScan> getActiveScans(String tserver) throws AccumuloException, AccumuloSecurityException;
/**
* List the active compaction running on a tablet server
@@@ -112,20 -104,16 +104,16 @@@
*
* @param tserver
* The tablet server address should be of the form <ip address>:<port>
- * @throws AccumuloException
* @since 1.5.0
*/
- public void ping(String tserver) throws AccumuloException;
+ void ping(String tserver) throws AccumuloException;
/**
* Test to see if the instance can load the given class as the given type. This check does not consider per table classpaths, see
* {@link TableOperations#testClassLoad(String, String, String)}
*
- * @param className
- * @param asTypeName
* @return true if the instance can load the given class as the given type, false otherwise
- * @throws AccumuloException
*/
- public boolean testClassLoad(final String className, final String asTypeName) throws AccumuloException, AccumuloSecurityException;
+ boolean testClassLoad(final String className, final String asTypeName) throws AccumuloException, AccumuloSecurityException;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
index 6166673,0823656..d9919ef
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
@@@ -109,13 -109,10 +109,10 @@@ public interface TableOperations
* Name of a table to create and import into.
* @param importDir
* Directory that contains the files copied by distcp from exportTable
- * @throws TableExistsException
- * @throws AccumuloException
- * @throws AccumuloSecurityException
* @since 1.5.0
*/
- public void importTable(String tableName, String importDir) throws TableExistsException, AccumuloException, AccumuloSecurityException;
-
+ void importTable(String tableName, String importDir) throws TableExistsException, AccumuloException, AccumuloSecurityException;
+
/**
* Exports a table. The tables data is not exported, just table metadata and a list of files to distcp. The table being exported must be offline and stay
* offline for the duration of distcp. To avoid losing access to a table it can be cloned and the clone taken offline for export.
@@@ -127,12 -124,9 +124,9 @@@
* Name of the table to export.
* @param exportDir
* An empty directory in HDFS where files containing table metadata and list of files to distcp will be placed.
- * @throws TableNotFoundException
- * @throws AccumuloException
- * @throws AccumuloSecurityException
* @since 1.5.0
*/
- public void exportTable(String tableName, String exportDir) throws TableNotFoundException, AccumuloException, AccumuloSecurityException;
+ void exportTable(String tableName, String exportDir) throws TableNotFoundException, AccumuloException, AccumuloSecurityException;
/**
* Ensures that tablets are split along a set of keys.
@@@ -212,11 -205,10 +205,10 @@@
* @throws AccumuloSecurityException
* if the user does not have permission
* @return the split points (end-row names) for the table's current split profile, grouped into fewer splits so as not to exceed maxSplits
- * @throws TableNotFoundException
* @since 1.5.0
*/
- public Collection<Text> listSplits(String tableName, int maxSplits) throws TableNotFoundException, AccumuloSecurityException, AccumuloException;
-
+ Collection<Text> listSplits(String tableName, int maxSplits) throws TableNotFoundException, AccumuloSecurityException, AccumuloException;
+
/**
* Finds the max row within a given range. To find the max row in a table, pass null for start and end row.
*
@@@ -233,14 -224,10 +224,10 @@@
* determines if the end row is included
*
* @return The max row in the range, or null if there is no visible data in the range.
- *
- * @throws AccumuloSecurityException
- * @throws AccumuloException
- * @throws TableNotFoundException
*/
- public Text getMaxRow(String tableName, Authorizations auths, Text startRow, boolean startInclusive, Text endRow, boolean endInclusive)
+ Text getMaxRow(String tableName, Authorizations auths, Text startRow, boolean startInclusive, Text endRow, boolean endInclusive)
throws TableNotFoundException, AccumuloException, AccumuloSecurityException;
-
+
/**
* Merge tablets between (start, end]
*
@@@ -401,10 -388,9 +388,9 @@@
* if a general error occurs
* @throws AccumuloSecurityException
* if the user does not have permission
- * @throws TableNotFoundException
*/
- public void flush(String tableName, Text start, Text end, boolean wait) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
-
+ void flush(String tableName, Text start, Text end, boolean wait) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
/**
* Sets a property on a table. Note that it may take a short period of time (a second) to propagate the change everywhere.
*
@@@ -527,10 -513,9 +513,9 @@@
* when there is a general accumulo error
* @throws AccumuloSecurityException
* when the user does not have the proper permissions
- * @throws TableNotFoundException
*/
- public void offline(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
-
+ void offline(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
+
/**
*
* @param tableName
@@@ -555,25 -524,9 +540,24 @@@
* when there is a general accumulo error
* @throws AccumuloSecurityException
* when the user does not have the proper permissions
- * @throws TableNotFoundException
*/
- public void online(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
-
+ void online(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
+
+ /**
+ *
+ * @param tableName
+ * the table to take online
+ * @param wait
+ * if true, then will not return until table is online
+ * @throws AccumuloException
+ * when there is a general accumulo error
+ * @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
+ * @throws TableNotFoundException
+ * @since 1.6.0
+ */
+ void online(String tableName, boolean wait) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
+
/**
* Clears the tablet locator cache for a specified table
*
@@@ -669,12 -618,9 +649,9 @@@
* @param tableName
* the name of the table
* @return a set of iterator names
- * @throws AccumuloSecurityException
- * @throws AccumuloException
- * @throws TableNotFoundException
*/
- public Map<String,EnumSet<IteratorScope>> listIterators(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
-
+ Map<String,EnumSet<IteratorScope>> listIterators(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
+
/**
* Check whether a given iterator configuration conflicts with existing configuration; in particular, determine if the name or priority are already in use for
* the specified scopes.
@@@ -700,11 -646,10 +677,10 @@@
* thrown if the constraint has already been added to the table or if there are errors in the configuration of existing constraints
* @throws AccumuloSecurityException
* thrown if the user doesn't have permission to add the constraint
- * @throws TableNotFoundException
* @since 1.5.0
*/
- public int addConstraint(String tableName, String constraintClassName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
-
+ int addConstraint(String tableName, String constraintClassName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
/**
* Remove a constraint from a table.
*
@@@ -727,23 -671,10 +702,22 @@@
* @return a map from constraint class name to assigned number
* @throws AccumuloException
* thrown if there are errors in the configuration of existing constraints
- * @throws TableNotFoundException
* @since 1.5.0
*/
- public Map<String,Integer> listConstraints(String tableName) throws AccumuloException, TableNotFoundException;
-
+ Map<String,Integer> listConstraints(String tableName) throws AccumuloException, TableNotFoundException;
+
+ /**
+ * Gets the number of bytes being used in the files for a set of tables
+ *
+ * @param tables
+ * a set of tables
+ * @return a list of disk usage objects containing linked table names and sizes
+ * @throws AccumuloException
+ * @throws AccumuloSecurityException
+ * @since 1.6.0
+ */
+ List<DiskUsage> getDiskUsage(Set<String> tables) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
/**
* Test to see if the instance can load the given class as the given type. This check uses the table classpath if it is set.
*
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
index 9d033e2,442f1be..3d69cc1
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
@@@ -1316,14 -1129,12 +1314,13 @@@ public class TableOperationsImpl extend
* when there is a general accumulo error
* @throws AccumuloSecurityException
* when the user does not have the proper permissions
- * @throws TableNotFoundException
*/
@Override
- public void offline(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
+ public void offline(String tableName, boolean wait) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
ArgumentChecker.notNull(tableName);
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)));
+ String tableId = Tables.getTableId(instance, tableName);
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)));
Map<String,String> opts = new HashMap<String,String>();
try {
@@@ -1350,13 -1153,11 +1347,12 @@@
* when there is a general accumulo error
* @throws AccumuloSecurityException
* when the user does not have the proper permissions
- * @throws TableNotFoundException
*/
@Override
- public void online(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
+ public void online(String tableName, boolean wait) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
ArgumentChecker.notNull(tableName);
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)));
+ String tableId = Tables.getTableId(instance, tableName);
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)));
Map<String,String> opts = new HashMap<String,String>();
try {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftTransportPool.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java
index ef7bcab,0000000..dad62ca
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java
@@@ -1,649 -1,0 +1,647 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapred;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.ClientSideIteratorScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableDeletedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.client.impl.OfflineScanner;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.TabletLocator;
+import org.apache.accumulo.core.client.mapreduce.InputTableConfig;
+import org.apache.accumulo.core.client.mapreduce.lib.util.InputConfigurator;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * An abstract input format to provide shared methods common to all other input format classes. At the very least, any classes inheriting from this class will
+ * need to define their own {@link RecordReader}.
+ */
+public abstract class AbstractInputFormat<K,V> implements InputFormat<K,V> {
+ protected static final Class<?> CLASS = AccumuloInputFormat.class;
+ protected static final Logger log = Logger.getLogger(CLASS);
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
+ * conversion to a string, and is not intended to be secure.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission)
+ * @param token
+ * the user's password
- * @throws org.apache.accumulo.core.client.AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public static void setConnectorInfo(JobConf job, String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ InputConfigurator.setConnectorInfo(CLASS, job, principal, token);
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * Stores the password in a file in HDFS and pulls that into the Distributed Cache in an attempt to be more secure than storing it in the Configuration.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission)
+ * @param tokenFile
+ * the path to the token file
- * @throws AccumuloSecurityException
+ * @since 1.6.0
+ */
+ public static void setConnectorInfo(JobConf job, String principal, String tokenFile) throws AccumuloSecurityException {
+ InputConfigurator.setConnectorInfo(CLASS, job, principal, tokenFile);
+ }
+
+ /**
+ * Determines if the connector has been configured.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the connector has been configured, false otherwise
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static Boolean isConnectorInfoSet(JobConf job) {
+ return InputConfigurator.isConnectorInfoSet(CLASS, job);
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static String getPrincipal(JobConf job) {
+ return InputConfigurator.getPrincipal(CLASS, job);
+ }
+
+ /**
+ * Gets the serialized token class from either the configuration or the token file.
+ *
+ * @since 1.5.0
+ * @deprecated since 1.6.0; Use {@link #getAuthenticationToken(JobConf)} instead.
+ */
+ @Deprecated
+ protected static String getTokenClass(JobConf job) {
+ return getAuthenticationToken(job).getClass().getName();
+ }
+
+ /**
+ * Gets the serialized token from either the configuration or the token file.
+ *
+ * @since 1.5.0
+ * @deprecated since 1.6.0; Use {@link #getAuthenticationToken(JobConf)} instead.
+ */
+ @Deprecated
+ protected static byte[] getToken(JobConf job) {
+ return AuthenticationToken.AuthenticationTokenSerializer.serialize(getAuthenticationToken(job));
+ }
+
+ /**
+ * Gets the authenticated token from either the specified token file or directly from the configuration, whichever was used when the job was configured.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the principal's authentication token
+ * @since 1.6.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ * @see #setConnectorInfo(JobConf, String, String)
+ */
+ protected static AuthenticationToken getAuthenticationToken(JobConf job) {
+ return InputConfigurator.getAuthenticationToken(CLASS, job);
+ }
+
+ /**
+ * Configures a {@link org.apache.accumulo.core.client.ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @param zooKeepers
+ * a comma-separated list of zookeeper servers
+ * @since 1.5.0
+ * @deprecated since 1.6.0; Use {@link #setZooKeeperInstance(JobConf, ClientConfiguration)} instead.
+ */
+ @Deprecated
+ public static void setZooKeeperInstance(JobConf job, String instanceName, String zooKeepers) {
+ InputConfigurator.setZooKeeperInstance(CLASS, job, instanceName, zooKeepers);
+ }
+
+ /**
+ * Configures a {@link org.apache.accumulo.core.client.ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param clientConfig
+ * client configuration containing connection options
+ * @since 1.6.0
+ */
+ public static void setZooKeeperInstance(JobConf job, ClientConfiguration clientConfig) {
+ InputConfigurator.setZooKeeperInstance(CLASS, job, clientConfig);
+ }
+
+ /**
+ * Configures a {@link org.apache.accumulo.core.client.mock.MockInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @since 1.5.0
+ */
+ public static void setMockInstance(JobConf job, String instanceName) {
+ InputConfigurator.setMockInstance(CLASS, job, instanceName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link org.apache.accumulo.core.client.Instance} based on the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return an Accumulo instance
+ * @since 1.5.0
+ * @see #setZooKeeperInstance(JobConf, String, String)
+ * @see #setMockInstance(JobConf, String)
+ */
+ protected static Instance getInstance(JobConf job) {
+ return InputConfigurator.getInstance(CLASS, job);
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param level
+ * the logging level
+ * @since 1.5.0
+ */
+ public static void setLogLevel(JobConf job, Level level) {
+ InputConfigurator.setLogLevel(CLASS, job, level);
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the log level
+ * @since 1.5.0
+ * @see #setLogLevel(JobConf, Level)
+ */
+ protected static Level getLogLevel(JobConf job) {
+ return InputConfigurator.getLogLevel(CLASS, job);
+ }
+
+ /**
+ * Sets the {@link org.apache.accumulo.core.security.Authorizations} used to scan. Must be a subset of the user's authorization. Defaults to the empty set.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param auths
+ * the user's authorizations
+ * @since 1.5.0
+ */
+ public static void setScanAuthorizations(JobConf job, Authorizations auths) {
+ InputConfigurator.setScanAuthorizations(CLASS, job, auths);
+ }
+
+ /**
+ * Gets the authorizations to set for the scans from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the Accumulo scan authorizations
+ * @since 1.5.0
+ * @see #setScanAuthorizations(JobConf, Authorizations)
+ */
+ protected static Authorizations getScanAuthorizations(JobConf job) {
+ return InputConfigurator.getScanAuthorizations(CLASS, job);
+ }
+
+ /**
+ * Initializes an Accumulo {@link org.apache.accumulo.core.client.impl.TabletLocator} based on the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return an Accumulo tablet locator
+ * @throws org.apache.accumulo.core.client.TableNotFoundException
+ * if the table name set on the configuration doesn't exist
+ * @since 1.6.0
+ */
+ protected static TabletLocator getTabletLocator(JobConf job, String tableId) throws TableNotFoundException {
+ return InputConfigurator.getTabletLocator(CLASS, job, tableId);
+ }
+
+ // InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext job)
+ /**
+ * Check whether a configuration is fully configured to be used with an Accumulo {@link InputFormat}.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @throws java.io.IOException
+ * if the context is improperly configured
+ * @since 1.5.0
+ */
+ protected static void validateOptions(JobConf job) throws IOException {
+ InputConfigurator.validateOptions(CLASS, job);
+ }
+
+ /**
+ * Fetches all {@link InputTableConfig}s that have been set on the given Hadoop job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @return the {@link InputTableConfig} objects set on the job
+ * @since 1.6.0
+ */
+ public static Map<String,InputTableConfig> getInputTableConfigs(JobConf job) {
+ return InputConfigurator.getInputTableConfigs(CLASS, job);
+ }
+
+ /**
+ * Fetches a {@link InputTableConfig} that has been set on the configuration for a specific table.
+ *
+ * <p>
+ * null is returned in the event that the table doesn't exist.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param tableName
+ * the table name for which to grab the config object
+ * @return the {@link InputTableConfig} for the given table
+ * @since 1.6.0
+ */
+ public static InputTableConfig getInputTableConfig(JobConf job, String tableName) {
+ return InputConfigurator.getInputTableConfig(CLASS, job, tableName);
+ }
+
+ /**
+ * An abstract base class to be used to create {@link org.apache.hadoop.mapred.RecordReader} instances that convert from Accumulo
+ * {@link org.apache.accumulo.core.data.Key}/{@link org.apache.accumulo.core.data.Value} pairs to the user's K/V types.
+ *
+ * Subclasses must implement {@link #next(Object, Object)} to update key and value, and also to update the following variables:
+ * <ul>
+ * <li>Key {@link #currentKey} (used for progress reporting)</li>
+ * <li>int {@link #numKeysRead} (used for progress reporting)</li>
+ * </ul>
+ */
+ protected abstract static class AbstractRecordReader<K,V> implements RecordReader<K,V> {
+ protected long numKeysRead;
+ protected Iterator<Map.Entry<Key,Value>> scannerIterator;
+ protected RangeInputSplit split;
+
+ /**
+ * Configures the iterators on a scanner for the given table name.
+ *
+ * @param job
+ * the Hadoop job configuration
+ * @param scanner
+ * the scanner for which to configure the iterators
+ * @param tableName
+ * the table name for which the scanner is configured
+ * @since 1.6.0
+ */
+ protected abstract void setupIterators(JobConf job, Scanner scanner, String tableName, RangeInputSplit split);
+
+ /**
+ * Initialize a scanner over the given input split using this task attempt configuration.
+ */
+ public void initialize(InputSplit inSplit, JobConf job) throws IOException {
+ Scanner scanner;
+ split = (RangeInputSplit) inSplit;
+ log.debug("Initializing input split: " + split.getRange());
+
+ Instance instance = split.getInstance();
+ if (null == instance) {
+ instance = getInstance(job);
+ }
+
+ String principal = split.getPrincipal();
+ if (null == principal) {
+ principal = getPrincipal(job);
+ }
+
+ AuthenticationToken token = split.getToken();
+ if (null == token) {
+ token = getAuthenticationToken(job);
+ }
+
+ Authorizations authorizations = split.getAuths();
+ if (null == authorizations) {
+ authorizations = getScanAuthorizations(job);
+ }
+
+ String table = split.getTableName();
+
+ // in case the table name changed, we can still use the previous name for terms of configuration,
+ // but the scanner will use the table id resolved at job setup time
+ InputTableConfig tableConfig = getInputTableConfig(job, split.getTableName());
+
+ Boolean isOffline = split.isOffline();
+ if (null == isOffline) {
+ isOffline = tableConfig.isOfflineScan();
+ }
+
+ Boolean isIsolated = split.isIsolatedScan();
+ if (null == isIsolated) {
+ isIsolated = tableConfig.shouldUseIsolatedScanners();
+ }
+
+ Boolean usesLocalIterators = split.usesLocalIterators();
+ if (null == usesLocalIterators) {
+ usesLocalIterators = tableConfig.shouldUseLocalIterators();
+ }
+
+ List<IteratorSetting> iterators = split.getIterators();
+ if (null == iterators) {
+ iterators = tableConfig.getIterators();
+ }
+
+ Collection<Pair<Text,Text>> columns = split.getFetchedColumns();
+ if (null == columns) {
+ columns = tableConfig.getFetchedColumns();
+ }
+
+ try {
+ log.debug("Creating connector with user: " + principal);
+ log.debug("Creating scanner for table: " + table);
+ log.debug("Authorizations are: " + authorizations);
+ if (isOffline) {
+ scanner = new OfflineScanner(instance, new Credentials(principal, token), split.getTableId(), authorizations);
+ } else if (instance instanceof MockInstance) {
+ scanner = instance.getConnector(principal, token).createScanner(split.getTableName(), authorizations);
+ } else {
+ scanner = new ScannerImpl(instance, new Credentials(principal, token), split.getTableId(), authorizations);
+ }
+ if (isIsolated) {
+ log.info("Creating isolated scanner");
+ scanner = new IsolatedScanner(scanner);
+ }
+ if (usesLocalIterators) {
+ log.info("Using local iterators");
+ scanner = new ClientSideIteratorScanner(scanner);
+ }
+ setupIterators(job, scanner, split.getTableName(), split);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+
+ // setup a scanner within the bounds of this split
+ for (Pair<Text,Text> c : columns) {
+ if (c.getSecond() != null) {
+ log.debug("Fetching column " + c.getFirst() + ":" + c.getSecond());
+ scanner.fetchColumn(c.getFirst(), c.getSecond());
+ } else {
+ log.debug("Fetching column family " + c.getFirst());
+ scanner.fetchColumnFamily(c.getFirst());
+ }
+ }
+
+ scanner.setRange(split.getRange());
+
+ numKeysRead = 0;
+
+ // do this last after setting all scanner options
+ scannerIterator = scanner.iterator();
+ }
+
+ @Override
+ public void close() {}
+
+ @Override
+ public long getPos() throws IOException {
+ return numKeysRead;
+ }
+
+ @Override
+ public float getProgress() throws IOException {
+ if (numKeysRead > 0 && currentKey == null)
+ return 1.0f;
+ return split.getProgress(currentKey);
+ }
+
+ protected Key currentKey = null;
+
+ }
+
+ Map<String,Map<KeyExtent,List<Range>>> binOfflineTable(JobConf job, String tableId, List<Range> ranges) throws TableNotFoundException, AccumuloException,
+ AccumuloSecurityException {
+
+ Instance instance = getInstance(job);
+ Connector conn = instance.getConnector(getPrincipal(job), getAuthenticationToken(job));
+
+ return InputConfigurator.binOffline(tableId, ranges, instance, conn);
+ }
+
+ /**
+ * Read the metadata table to get tablets and match up ranges to them.
+ */
+ @Override
+ public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
+ Level logLevel = getLogLevel(job);
+ log.setLevel(logLevel);
+ validateOptions(job);
+
+ Random random = new Random();
+ LinkedList<InputSplit> splits = new LinkedList<InputSplit>();
+ Map<String,InputTableConfig> tableConfigs = getInputTableConfigs(job);
+ for (Map.Entry<String,InputTableConfig> tableConfigEntry : tableConfigs.entrySet()) {
+ String tableName = tableConfigEntry.getKey();
+ InputTableConfig tableConfig = tableConfigEntry.getValue();
+
+ Instance instance = getInstance(job);
+ boolean mockInstance;
+ String tableId;
+ // resolve table name to id once, and use id from this point forward
+ if (instance instanceof MockInstance) {
+ tableId = "";
+ mockInstance = true;
+ } else {
+ try {
+ tableId = Tables.getTableId(instance, tableName);
+ } catch (TableNotFoundException e) {
+ throw new IOException(e);
+ }
+ mockInstance = false;
+ }
+
+ Authorizations auths = getScanAuthorizations(job);
+ String principal = getPrincipal(job);
+ AuthenticationToken token = getAuthenticationToken(job);
+
+ boolean autoAdjust = tableConfig.shouldAutoAdjustRanges();
+ List<Range> ranges = autoAdjust ? Range.mergeOverlapping(tableConfig.getRanges()) : tableConfig.getRanges();
+ if (ranges.isEmpty()) {
+ ranges = new ArrayList<Range>(1);
+ ranges.add(new Range());
+ }
+
+ // get the metadata information for these ranges
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+ TabletLocator tl;
+ try {
+ if (tableConfig.isOfflineScan()) {
+ binnedRanges = binOfflineTable(job, tableId, ranges);
+ while (binnedRanges == null) {
+ // Some tablets were still online, try again
+ UtilWaitThread.sleep(100 + random.nextInt(100)); // sleep randomly between 100 and 200 ms
+ binnedRanges = binOfflineTable(job, tableId, ranges);
+ }
+ } else {
+ tl = getTabletLocator(job, tableId);
+ // its possible that the cache could contain complete, but old information about a tables tablets... so clear it
+ tl.invalidateCache();
+ Credentials creds = new Credentials(getPrincipal(job), getAuthenticationToken(job));
+
+ while (!tl.binRanges(creds, ranges, binnedRanges).isEmpty()) {
+ if (!(instance instanceof MockInstance)) {
+ if (!Tables.exists(instance, tableId))
+ throw new TableDeletedException(tableId);
+ if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
+ throw new TableOfflineException(instance, tableId);
+ }
+ binnedRanges.clear();
+ log.warn("Unable to locate bins for specified ranges. Retrying.");
+ UtilWaitThread.sleep(100 + random.nextInt(100)); // sleep randomly between 100 and 200 ms
+ tl.invalidateCache();
+ }
+ }
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ HashMap<Range,ArrayList<String>> splitsToAdd = null;
+
+ if (!autoAdjust)
+ splitsToAdd = new HashMap<Range,ArrayList<String>>();
+
+ HashMap<String,String> hostNameCache = new HashMap<String,String>();
+ for (Map.Entry<String,Map<KeyExtent,List<Range>>> tserverBin : binnedRanges.entrySet()) {
+ String ip = tserverBin.getKey().split(":", 2)[0];
+ String location = hostNameCache.get(ip);
+ if (location == null) {
+ InetAddress inetAddress = InetAddress.getByName(ip);
+ location = inetAddress.getCanonicalHostName();
+ hostNameCache.put(ip, location);
+ }
+ for (Map.Entry<KeyExtent,List<Range>> extentRanges : tserverBin.getValue().entrySet()) {
+ Range ke = extentRanges.getKey().toDataRange();
+ for (Range r : extentRanges.getValue()) {
+ if (autoAdjust) {
+ // divide ranges into smaller ranges, based on the tablets
+ RangeInputSplit split = new RangeInputSplit(tableName, tableId, ke.clip(r), new String[] {location});
+
+ split.setOffline(tableConfig.isOfflineScan());
+ split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
+ split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());
+ split.setMockInstance(mockInstance);
+ split.setFetchedColumns(tableConfig.getFetchedColumns());
+ split.setPrincipal(principal);
+ split.setToken(token);
+ split.setInstanceName(instance.getInstanceName());
+ split.setZooKeepers(instance.getZooKeepers());
+ split.setAuths(auths);
+ split.setIterators(tableConfig.getIterators());
+ split.setLogLevel(logLevel);
+
+ splits.add(split);
+ } else {
+ // don't divide ranges
+ ArrayList<String> locations = splitsToAdd.get(r);
+ if (locations == null)
+ locations = new ArrayList<String>(1);
+ locations.add(location);
+ splitsToAdd.put(r, locations);
+ }
+ }
+ }
+ }
+
+ if (!autoAdjust)
+ for (Map.Entry<Range,ArrayList<String>> entry : splitsToAdd.entrySet()) {
+ RangeInputSplit split = new RangeInputSplit(tableName, tableId, entry.getKey(), entry.getValue().toArray(new String[0]));
+
+ split.setOffline(tableConfig.isOfflineScan());
+ split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
+ split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());
+ split.setMockInstance(mockInstance);
+ split.setFetchedColumns(tableConfig.getFetchedColumns());
+ split.setPrincipal(principal);
+ split.setToken(token);
+ split.setInstanceName(instance.getInstanceName());
+ split.setZooKeepers(instance.getZooKeepers());
+ split.setAuths(auths);
+ split.setIterators(tableConfig.getIterators());
+ split.setLogLevel(logLevel);
+
+ splits.add(split);
+ }
+ }
+
+ return splits.toArray(new InputSplit[splits.size()]);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
index 02512a4,d7be37c..1ec4c41
--- a/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
@@@ -91,26 -88,7 +90,25 @@@ public class AccumuloOutputFormat imple
public static void setConnectorInfo(JobConf job, String principal, AuthenticationToken token) throws AccumuloSecurityException {
OutputConfigurator.setConnectorInfo(CLASS, job, principal, token);
}
-
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * Stores the password in a file in HDFS and pulls that into the Distributed Cache in an attempt to be more secure than storing it in the Configuration.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission if {@link #setCreateTables(JobConf, boolean)} is set to true)
+ * @param tokenFile
+ * the path to the password file
- * @throws AccumuloSecurityException
+ * @since 1.6.0
+ */
+ public static void setConnectorInfo(JobConf job, String principal, String tokenFile) throws AccumuloSecurityException {
+ OutputConfigurator.setConnectorInfo(CLASS, job, principal, tokenFile);
+ }
+
/**
* Determines if the connector has been configured.
*
[58/64] [abbrv] Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/CompactionStrategy.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/CompactionStrategy.java
index 52688cb,0000000..7bc1a80
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/CompactionStrategy.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/CompactionStrategy.java
@@@ -1,71 -1,0 +1,67 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver.compaction;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * The interface for customizing major compactions.
+ * <p>
+ * The tablet server has one thread to ask many tablets if they should compact. When the strategy returns true, then tablet is added to the queue of tablets
+ * waiting for a compaction thread. Once a thread is available, the {@link #gatherInformation(MajorCompactionRequest)} method is called outside the tablets'
+ * lock. This gives the strategy the ability to read information that maybe expensive to fetch. Once the gatherInformation returns, the tablet lock is grabbed
+ * and the compactionPlan computed. This should *not* do expensive operations, especially not I/O. Note that the number of files may change between calls to
+ * {@link #gatherInformation(MajorCompactionRequest)} and {@link #getCompactionPlan(MajorCompactionRequest)}.
+ * <p>
+ * <b>Note:</b> the strategy object used for the {@link #shouldCompact(MajorCompactionRequest)} call is going to be different from the one used in the
+ * compaction thread.
+ */
+public abstract class CompactionStrategy {
+
+ /**
+ * The settings for the compaction strategy pulled from zookeeper. The <tt>table.compacations.major.strategy.opts</tt> part of the setting will be removed.
- *
- * @param options
+ */
+ public void init(Map<String,String> options) {}
+
+ /**
+ * Determine if this tablet is eligible for a major compaction. It's ok if it later determines (through {@link #gatherInformation(MajorCompactionRequest)} and
+ * {@link #getCompactionPlan(MajorCompactionRequest)}) that it does not need to. Any state stored during shouldCompact will no longer exist when
+ * {@link #gatherInformation(MajorCompactionRequest)} and {@link #getCompactionPlan(MajorCompactionRequest)} are called.
+ *
+ */
+ public abstract boolean shouldCompact(MajorCompactionRequest request) throws IOException;
+
+ /**
+ * Called prior to obtaining the tablet lock, useful for examining metadata or indexes. State collected during this method will be available during the call
+ * the {@link #getCompactionPlan(MajorCompactionRequest)}.
+ *
+ * @param request
+ * basic details about the tablet
- * @throws IOException
+ */
+ public void gatherInformation(MajorCompactionRequest request) throws IOException {}
+
+ /**
+ * Get the plan for compacting a tablets files. Called while holding the tablet lock, so it should not be doing any blocking.
+ *
+ * @param request
+ * basic details about the tablet
+ * @return the plan for a major compaction, or null to cancel the compaction.
- * @throws IOException
+ */
+ abstract public CompactionPlan getCompactionPlan(MajorCompactionRequest request) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
index 25b8043,0000000..a1229e7
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
@@@ -1,170 -1,0 +1,169 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver.logger;
+
+import java.io.DataInputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.tserver.log.DfsLogger;
+import org.apache.accumulo.tserver.log.DfsLogger.DFSLoggerInputStreams;
+import org.apache.accumulo.tserver.log.MultiReader;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+
+public class LogReader {
+
+ static class Opts extends Help {
+ @Parameter(names = "-r", description = "print only mutations associated with the given row")
+ String row;
+ @Parameter(names = "-m", description = "limit the number of mutations printed per row")
+ int maxMutations = 5;
+ @Parameter(names = "-t", description = "print only mutations that fall within the given key extent")
+ String extent;
+ @Parameter(names = "-p", description = "search for a row that matches the given regex")
+ String regexp;
+ @Parameter(description = "<logfile> { <logfile> ... }")
+ List<String> files = new ArrayList<String>();
+ }
+
+ /**
+ * Dump a Log File (Map or Sequence) to stdout. Will read from HDFS or local file system.
+ *
+ * @param args
+ * - first argument is the file to print
- * @throws IOException
+ */
+ public static void main(String[] args) throws IOException {
+ Opts opts = new Opts();
+ opts.parseArgs(LogReader.class.getName(), args);
+ VolumeManager fs = VolumeManagerImpl.get();
+
+ Matcher rowMatcher = null;
+ KeyExtent ke = null;
+ Text row = null;
+ if (opts.files.isEmpty()) {
+ new JCommander(opts).usage();
+ return;
+ }
+ if (opts.row != null)
+ row = new Text(opts.row);
+ if (opts.extent != null) {
+ String sa[] = opts.extent.split(";");
+ ke = new KeyExtent(new Text(sa[0]), new Text(sa[1]), new Text(sa[2]));
+ }
+ if (opts.regexp != null) {
+ Pattern pattern = Pattern.compile(opts.regexp);
+ rowMatcher = pattern.matcher("");
+ }
+
+ Set<Integer> tabletIds = new HashSet<Integer>();
+
+ for (String file : opts.files) {
+
+ Path path = new Path(file);
+ LogFileKey key = new LogFileKey();
+ LogFileValue value = new LogFileValue();
+
+ if (fs.isFile(path)) {
+ // read log entries from a simple hdfs file
+ DFSLoggerInputStreams streams = DfsLogger.readHeaderAndReturnStream(fs, path, ServerConfiguration.getSiteConfiguration());
+ DataInputStream input = streams.getDecryptingInputStream();
+
+ try {
+ while (true) {
+ try {
+ key.readFields(input);
+ value.readFields(input);
+ } catch (EOFException ex) {
+ break;
+ }
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
+ }
+ } finally {
+ input.close();
+ }
+ } else {
+ // read the log entries sorted in a map file
+ MultiReader input = new MultiReader(fs, path);
+ while (input.next(key, value)) {
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
+ }
+ }
+ }
+ }
+
+ public static void printLogEvent(LogFileKey key, LogFileValue value, Text row, Matcher rowMatcher, KeyExtent ke, Set<Integer> tabletIds, int maxMutations) {
+
+ if (ke != null) {
+ if (key.event == LogEvents.DEFINE_TABLET) {
+ if (key.tablet.equals(ke)) {
+ tabletIds.add(key.tid);
+ } else {
+ return;
+ }
+ } else if (!tabletIds.contains(key.tid)) {
+ return;
+ }
+ }
+
+ if (row != null || rowMatcher != null) {
+ if (key.event == LogEvents.MUTATION || key.event == LogEvents.MANY_MUTATIONS) {
+ boolean found = false;
+ for (Mutation m : value.mutations) {
+ if (row != null && new Text(m.getRow()).equals(row)) {
+ found = true;
+ break;
+ }
+
+ if (rowMatcher != null) {
+ rowMatcher.reset(new String(m.getRow(), Constants.UTF8));
+ if (rowMatcher.matches()) {
+ found = true;
+ break;
+ }
+ }
+ }
+
+ if (!found)
+ return;
+ } else {
+ return;
+ }
+
+ }
+
+ System.out.println(key);
+ System.out.println(LogFileValue.format(value, maxMutations));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
----------------------------------------------------------------------
diff --cc start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
index b7d5fa9,277c741..745401b
--- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
+++ b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
@@@ -36,15 -30,16 +30,16 @@@ public class PostDelegatingVFSClassLoad
super(files, manager, parent);
}
+ @Override
protected synchronized Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
Class<?> c = findLoadedClass(name);
- if (c == null) {
- try {
- // try finding this class here instead of parent
- findClass(name);
- } catch (ClassNotFoundException e) {
-
- }
+ if (c != null)
+ return c;
+ try {
+ // try finding this class here instead of parent
+ return findClass(name);
+ } catch (ClassNotFoundException e) {
+
}
return super.loadClass(name, resolve);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java
----------------------------------------------------------------------
diff --cc start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java
index c9fd2f5,104ea09..2973750
--- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java
+++ b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java
@@@ -42,76 -42,36 +42,71 @@@ import org.apache.hadoop.fs.Path
*/
public class HdfsFileSystem extends AbstractFileSystem
{
- private static final Log log = LogFactory.getLog(HdfsFileSystem.class);
-
- private FileSystem fs;
-
- protected HdfsFileSystem(final FileName rootName, final FileSystemOptions fileSystemOptions)
+ private static final Log log = LogFactory.getLog(HdfsFileSystem.class);
+
+ private FileSystem fs;
+
- /**
- *
- * @param rootName
- * @param fileSystemOptions
- */
+ protected HdfsFileSystem(final FileName rootName, final FileSystemOptions fileSystemOptions)
+ {
+ super(rootName, null, fileSystemOptions);
+ }
+
+ /**
+ * @see org.apache.commons.vfs2.provider.AbstractFileSystem#addCapabilities(java.util.Collection)
+ */
+ @Override
+ protected void addCapabilities(final Collection<Capability> capabilities)
+ {
+ capabilities.addAll(HdfsFileProvider.CAPABILITIES);
+ }
+
+ /**
+ * @see org.apache.commons.vfs2.provider.AbstractFileSystem#close()
+ */
+ @Override
+ synchronized public void close()
+ {
+ try
{
- super(rootName, null, fileSystemOptions);
+ if (null != fs)
+ {
+ fs.close();
+ }
}
-
- /**
- * @see org.apache.commons.vfs2.provider.AbstractFileSystem#addCapabilities(java.util.Collection)
- */
- @Override
- protected void addCapabilities(final Collection<Capability> capabilities)
+ catch (final IOException e)
{
- capabilities.addAll(HdfsFileProvider.CAPABILITIES);
+ throw new RuntimeException("Error closing HDFS client", e);
}
-
- /**
- * @see org.apache.commons.vfs2.provider.AbstractFileSystem#close()
- */
- @Override
- public void close()
+ super.close();
+ }
+
+ /**
+ * @see org.apache.commons.vfs2.provider.AbstractFileSystem#createFile(org.apache.commons.vfs2.provider.AbstractFileName)
+ */
+ @Override
+ protected FileObject createFile(final AbstractFileName name) throws Exception
+ {
+ throw new FileSystemException("Operation not supported");
+ }
+
+ /**
+ * @see org.apache.commons.vfs2.provider.AbstractFileSystem#resolveFile(org.apache.commons.vfs2.FileName)
+ */
+ @Override
+ public FileObject resolveFile(final FileName name) throws FileSystemException
+ {
+
+ synchronized (this)
{
+ if (null == this.fs)
+ {
+ final String hdfsUri = name.getRootURI();
+ final Configuration conf = new Configuration(true);
+ conf.set(org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY, hdfsUri);
+ this.fs = null;
try
{
- if (null != fs)
- {
- fs.close();
- }
+ fs = org.apache.hadoop.fs.FileSystem.get(conf);
}
catch (final IOException e)
{
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/test/src/main/java/org/apache/accumulo/test/functional/CacheTestClean.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
----------------------------------------------------------------------
[62/64] [abbrv] Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
index 5c7b857,0000000..53abbbe
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
@@@ -1,690 -1,0 +1,689 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.ClientSideIteratorScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableDeletedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.client.impl.OfflineScanner;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.TabletLocator;
+import org.apache.accumulo.core.client.mapreduce.lib.util.InputConfigurator;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * An abstract input format to provide shared methods common to all other input format classes. At the very least, any classes inheriting from this class will
+ * need to define their own {@link RecordReader}.
+ */
+public abstract class AbstractInputFormat<K,V> extends InputFormat<K,V> {
+
+ protected static final Class<?> CLASS = AccumuloInputFormat.class;
+ protected static final Logger log = Logger.getLogger(CLASS);
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
+ * conversion to a string, and is not intended to be secure.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission)
+ * @param token
+ * the user's password
- * @throws org.apache.accumulo.core.client.AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public static void setConnectorInfo(Job job, String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ InputConfigurator.setConnectorInfo(CLASS, job.getConfiguration(), principal, token);
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * Stores the password in a file in HDFS and pulls that into the Distributed Cache in an attempt to be more secure than storing it in the Configuration.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission)
+ * @param tokenFile
+ * the path to the token file
- * @throws AccumuloSecurityException
+ * @since 1.6.0
+ */
+ public static void setConnectorInfo(Job job, String principal, String tokenFile) throws AccumuloSecurityException {
+ InputConfigurator.setConnectorInfo(CLASS, job.getConfiguration(), principal, tokenFile);
+ }
+
+ /**
+ * Determines if the connector has been configured.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the connector has been configured, false otherwise
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static Boolean isConnectorInfoSet(JobContext context) {
+ return InputConfigurator.isConnectorInfoSet(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static String getPrincipal(JobContext context) {
+ return InputConfigurator.getPrincipal(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Gets the serialized token class from either the configuration or the token file.
+ *
+ * @since 1.5.0
+ * @deprecated since 1.6.0; Use {@link #getAuthenticationToken(JobContext)} instead.
+ */
+ @Deprecated
+ protected static String getTokenClass(JobContext context) {
+ return getAuthenticationToken(context).getClass().getName();
+ }
+
+ /**
+ * Gets the serialized token from either the configuration or the token file.
+ *
+ * @since 1.5.0
+ * @deprecated since 1.6.0; Use {@link #getAuthenticationToken(JobContext)} instead.
+ */
+ @Deprecated
+ protected static byte[] getToken(JobContext context) {
+ return AuthenticationToken.AuthenticationTokenSerializer.serialize(getAuthenticationToken(context));
+ }
+
+ /**
+ * Gets the authenticated token from either the specified token file or directly from the configuration, whichever was used when the job was configured.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the principal's authentication token
+ * @since 1.6.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ * @see #setConnectorInfo(Job, String, String)
+ */
+ protected static AuthenticationToken getAuthenticationToken(JobContext context) {
+ return InputConfigurator.getAuthenticationToken(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Configures a {@link org.apache.accumulo.core.client.ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @param zooKeepers
+ * a comma-separated list of zookeeper servers
+ * @since 1.5.0
+ * @deprecated since 1.6.0; Use {@link #setZooKeeperInstance(Job, ClientConfiguration)} instead.
+ */
+ @Deprecated
+ public static void setZooKeeperInstance(Job job, String instanceName, String zooKeepers) {
+ InputConfigurator.setZooKeeperInstance(CLASS, job.getConfiguration(), instanceName, zooKeepers);
+ }
+
+ /**
+ * Configures a {@link org.apache.accumulo.core.client.ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param clientConfig
+ * client configuration containing connection options
+ * @since 1.6.0
+ */
+ public static void setZooKeeperInstance(Job job, ClientConfiguration clientConfig) {
+ InputConfigurator.setZooKeeperInstance(CLASS, job.getConfiguration(), clientConfig);
+ }
+
+ /**
+ * Configures a {@link org.apache.accumulo.core.client.mock.MockInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @since 1.5.0
+ */
+ public static void setMockInstance(Job job, String instanceName) {
+ InputConfigurator.setMockInstance(CLASS, job.getConfiguration(), instanceName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link org.apache.accumulo.core.client.Instance} based on the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return an Accumulo instance
+ * @since 1.5.0
+ * @see #setZooKeeperInstance(Job, String, String)
+ * @see #setMockInstance(Job, String)
+ */
+ protected static Instance getInstance(JobContext context) {
+ return InputConfigurator.getInstance(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param level
+ * the logging level
+ * @since 1.5.0
+ */
+ public static void setLogLevel(Job job, Level level) {
+ InputConfigurator.setLogLevel(CLASS, job.getConfiguration(), level);
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the log level
+ * @since 1.5.0
+ * @see #setLogLevel(Job, Level)
+ */
+ protected static Level getLogLevel(JobContext context) {
+ return InputConfigurator.getLogLevel(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Sets the {@link org.apache.accumulo.core.security.Authorizations} used to scan. Must be a subset of the user's authorization. Defaults to the empty set.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param auths
+ * the user's authorizations
+ */
+ public static void setScanAuthorizations(Job job, Authorizations auths) {
+ InputConfigurator.setScanAuthorizations(CLASS, job.getConfiguration(), auths);
+ }
+
+ /**
+ * Gets the authorizations to set for the scans from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the Accumulo scan authorizations
+ * @since 1.5.0
+ * @see #setScanAuthorizations(Job, Authorizations)
+ */
+ protected static Authorizations getScanAuthorizations(JobContext context) {
+ return InputConfigurator.getScanAuthorizations(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Fetches all {@link InputTableConfig}s that have been set on the given job.
+ *
+ * @param context
+ * the Hadoop job instance to be configured
+ * @return the {@link InputTableConfig} objects for the job
+ * @since 1.6.0
+ */
+ protected static Map<String,InputTableConfig> getInputTableConfigs(JobContext context) {
+ return InputConfigurator.getInputTableConfigs(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Fetches a {@link InputTableConfig} that has been set on the configuration for a specific table.
+ *
+ * <p>
+ * null is returned in the event that the table doesn't exist.
+ *
+ * @param context
+ * the Hadoop job instance to be configured
+ * @param tableName
+ * the table name for which to grab the config object
+ * @return the {@link InputTableConfig} for the given table
+ * @since 1.6.0
+ */
+ protected static InputTableConfig getInputTableConfig(JobContext context, String tableName) {
+ return InputConfigurator.getInputTableConfig(CLASS, getConfiguration(context), tableName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link org.apache.accumulo.core.client.impl.TabletLocator} based on the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @param table
+ * the table for which to initialize the locator
+ * @return an Accumulo tablet locator
+ * @throws org.apache.accumulo.core.client.TableNotFoundException
+ * if the table name set on the configuration doesn't exist
+ * @since 1.6.0
+ */
+ protected static TabletLocator getTabletLocator(JobContext context, String table) throws TableNotFoundException {
+ return InputConfigurator.getTabletLocator(CLASS, getConfiguration(context), table);
+ }
+
+ // InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext job)
+ /**
+ * Check whether a configuration is fully configured to be used with an Accumulo {@link org.apache.hadoop.mapreduce.InputFormat}.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @throws java.io.IOException
+ * if the context is improperly configured
+ * @since 1.5.0
+ */
+ protected static void validateOptions(JobContext context) throws IOException {
+ InputConfigurator.validateOptions(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * An abstract base class to be used to create {@link org.apache.hadoop.mapreduce.RecordReader} instances that convert from Accumulo
+ * {@link org.apache.accumulo.core.data.Key}/{@link org.apache.accumulo.core.data.Value} pairs to the user's K/V types.
+ *
+ * Subclasses must implement {@link #nextKeyValue()} and use it to update the following variables:
+ * <ul>
+ * <li>K {@link #currentK}</li>
+ * <li>V {@link #currentV}</li>
+ * <li>Key {@link #currentKey} (used for progress reporting)</li>
+ * <li>int {@link #numKeysRead} (used for progress reporting)</li>
+ * </ul>
+ */
+ protected abstract static class AbstractRecordReader<K,V> extends RecordReader<K,V> {
+ protected long numKeysRead;
+ protected Iterator<Map.Entry<Key,Value>> scannerIterator;
+ protected RangeInputSplit split;
+
+ /**
+ * Configures the iterators on a scanner for the given table name.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @param scanner
+ * the scanner for which to configure the iterators
+ * @param tableName
+ * the table name for which the scanner is configured
+ * @since 1.6.0
+ */
+ protected abstract void setupIterators(TaskAttemptContext context, Scanner scanner, String tableName, RangeInputSplit split);
+
+ /**
+ * Initialize a scanner over the given input split using this task attempt configuration.
+ */
+ @Override
+ public void initialize(InputSplit inSplit, TaskAttemptContext attempt) throws IOException {
+
+ Scanner scanner;
+ split = (RangeInputSplit) inSplit;
+ log.debug("Initializing input split: " + split.getRange());
+
+ Instance instance = split.getInstance();
+ if (null == instance) {
+ instance = getInstance(attempt);
+ }
+
+ String principal = split.getPrincipal();
+ if (null == principal) {
+ principal = getPrincipal(attempt);
+ }
+
+ AuthenticationToken token = split.getToken();
+ if (null == token) {
+ token = getAuthenticationToken(attempt);
+ }
+
+ Authorizations authorizations = split.getAuths();
+ if (null == authorizations) {
+ authorizations = getScanAuthorizations(attempt);
+ }
+
+ String table = split.getTableName();
+
+ // in case the table name changed, we can still use the previous name for terms of configuration,
+ // but the scanner will use the table id resolved at job setup time
+ InputTableConfig tableConfig = getInputTableConfig(attempt, split.getTableName());
+
+ Boolean isOffline = split.isOffline();
+ if (null == isOffline) {
+ isOffline = tableConfig.isOfflineScan();
+ }
+
+ Boolean isIsolated = split.isIsolatedScan();
+ if (null == isIsolated) {
+ isIsolated = tableConfig.shouldUseIsolatedScanners();
+ }
+
+ Boolean usesLocalIterators = split.usesLocalIterators();
+ if (null == usesLocalIterators) {
+ usesLocalIterators = tableConfig.shouldUseLocalIterators();
+ }
+
+ List<IteratorSetting> iterators = split.getIterators();
+ if (null == iterators) {
+ iterators = tableConfig.getIterators();
+ }
+
+ Collection<Pair<Text,Text>> columns = split.getFetchedColumns();
+ if (null == columns) {
+ columns = tableConfig.getFetchedColumns();
+ }
+
+ try {
+ log.debug("Creating connector with user: " + principal);
+ log.debug("Creating scanner for table: " + table);
+ log.debug("Authorizations are: " + authorizations);
+ if (isOffline) {
+ scanner = new OfflineScanner(instance, new Credentials(principal, token), split.getTableId(), authorizations);
+ } else if (instance instanceof MockInstance) {
+ scanner = instance.getConnector(principal, token).createScanner(split.getTableName(), authorizations);
+ } else {
+ scanner = new ScannerImpl(instance, new Credentials(principal, token), split.getTableId(), authorizations);
+ }
+ if (isIsolated) {
+ log.info("Creating isolated scanner");
+ scanner = new IsolatedScanner(scanner);
+ }
+ if (usesLocalIterators) {
+ log.info("Using local iterators");
+ scanner = new ClientSideIteratorScanner(scanner);
+ }
+
+ setupIterators(attempt, scanner, split.getTableName(), split);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ // setup a scanner within the bounds of this split
+ for (Pair<Text,Text> c : columns) {
+ if (c.getSecond() != null) {
+ log.debug("Fetching column " + c.getFirst() + ":" + c.getSecond());
+ scanner.fetchColumn(c.getFirst(), c.getSecond());
+ } else {
+ log.debug("Fetching column family " + c.getFirst());
+ scanner.fetchColumnFamily(c.getFirst());
+ }
+ }
+
+ scanner.setRange(split.getRange());
+ numKeysRead = 0;
+
+ // do this last after setting all scanner options
+ scannerIterator = scanner.iterator();
+ }
+
+ @Override
+ public void close() {}
+
+ @Override
+ public float getProgress() throws IOException {
+ if (numKeysRead > 0 && currentKey == null)
+ return 1.0f;
+ return split.getProgress(currentKey);
+ }
+
+ /**
+ * The Key that should be returned to the client
+ */
+ protected K currentK = null;
+
+ /**
+ * The Value that should be return to the client
+ */
+ protected V currentV = null;
+
+ /**
+ * The Key that is used to determine progress in the current InputSplit. It is not returned to the client and is only used internally
+ */
+ protected Key currentKey = null;
+
+ @Override
+ public K getCurrentKey() throws IOException, InterruptedException {
+ return currentK;
+ }
+
+ @Override
+ public V getCurrentValue() throws IOException, InterruptedException {
+ return currentV;
+ }
+ }
+
+ Map<String,Map<KeyExtent,List<Range>>> binOfflineTable(JobContext context, String tableId, List<Range> ranges) throws TableNotFoundException,
+ AccumuloException, AccumuloSecurityException {
+
+ Instance instance = getInstance(context);
+ Connector conn = instance.getConnector(getPrincipal(context), getAuthenticationToken(context));
+
+ return InputConfigurator.binOffline(tableId, ranges, instance, conn);
+ }
+
+ /**
+ * Gets the splits of the tables that have been set on the job.
+ *
+ * @param context
+ * the configuration of the job
+ * @return the splits from the tables based on the ranges.
+ * @throws java.io.IOException
+ * if a table set on the job doesn't exist or an error occurs initializing the tablet locator
+ */
++ @Override
+ public List<InputSplit> getSplits(JobContext context) throws IOException {
+ Level logLevel = getLogLevel(context);
+ log.setLevel(logLevel);
+ validateOptions(context);
+ Random random = new Random();
+ LinkedList<InputSplit> splits = new LinkedList<InputSplit>();
+ Map<String,InputTableConfig> tableConfigs = getInputTableConfigs(context);
+ for (Map.Entry<String,InputTableConfig> tableConfigEntry : tableConfigs.entrySet()) {
+
+ String tableName = tableConfigEntry.getKey();
+ InputTableConfig tableConfig = tableConfigEntry.getValue();
+
+ Instance instance = getInstance(context);
+ boolean mockInstance;
+ String tableId;
+ // resolve table name to id once, and use id from this point forward
+ if (instance instanceof MockInstance) {
+ tableId = "";
+ mockInstance = true;
+ } else {
+ try {
+ tableId = Tables.getTableId(instance, tableName);
+ } catch (TableNotFoundException e) {
+ throw new IOException(e);
+ }
+ mockInstance = false;
+ }
+
+ Authorizations auths = getScanAuthorizations(context);
+ String principal = getPrincipal(context);
+ AuthenticationToken token = getAuthenticationToken(context);
+
+ boolean autoAdjust = tableConfig.shouldAutoAdjustRanges();
+ List<Range> ranges = autoAdjust ? Range.mergeOverlapping(tableConfig.getRanges()) : tableConfig.getRanges();
+ if (ranges.isEmpty()) {
+ ranges = new ArrayList<Range>(1);
+ ranges.add(new Range());
+ }
+
+ // get the metadata information for these ranges
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+ TabletLocator tl;
+ try {
+ if (tableConfig.isOfflineScan()) {
+ binnedRanges = binOfflineTable(context, tableId, ranges);
+ while (binnedRanges == null) {
+ // Some tablets were still online, try again
+ UtilWaitThread.sleep(100 + random.nextInt(100)); // sleep randomly between 100 and 200 ms
+ binnedRanges = binOfflineTable(context, tableId, ranges);
+
+ }
+ } else {
+ tl = getTabletLocator(context, tableId);
+ // its possible that the cache could contain complete, but old information about a tables tablets... so clear it
+ tl.invalidateCache();
+ Credentials creds = new Credentials(getPrincipal(context), getAuthenticationToken(context));
+
+ while (!tl.binRanges(creds, ranges, binnedRanges).isEmpty()) {
+ if (!(instance instanceof MockInstance)) {
+ if (!Tables.exists(instance, tableId))
+ throw new TableDeletedException(tableId);
+ if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
+ throw new TableOfflineException(instance, tableId);
+ }
+ binnedRanges.clear();
+ log.warn("Unable to locate bins for specified ranges. Retrying.");
+ UtilWaitThread.sleep(100 + random.nextInt(100)); // sleep randomly between 100 and 200 ms
+ tl.invalidateCache();
+ }
+ }
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ HashMap<Range,ArrayList<String>> splitsToAdd = null;
+
+ if (!autoAdjust)
+ splitsToAdd = new HashMap<Range,ArrayList<String>>();
+
+ HashMap<String,String> hostNameCache = new HashMap<String,String>();
+ for (Map.Entry<String,Map<KeyExtent,List<Range>>> tserverBin : binnedRanges.entrySet()) {
+ String ip = tserverBin.getKey().split(":", 2)[0];
+ String location = hostNameCache.get(ip);
+ if (location == null) {
+ InetAddress inetAddress = InetAddress.getByName(ip);
+ location = inetAddress.getCanonicalHostName();
+ hostNameCache.put(ip, location);
+ }
+ for (Map.Entry<KeyExtent,List<Range>> extentRanges : tserverBin.getValue().entrySet()) {
+ Range ke = extentRanges.getKey().toDataRange();
+ for (Range r : extentRanges.getValue()) {
+ if (autoAdjust) {
+ // divide ranges into smaller ranges, based on the tablets
+ RangeInputSplit split = new RangeInputSplit(tableName, tableId, ke.clip(r), new String[] {location});
+
+ split.setOffline(tableConfig.isOfflineScan());
+ split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
+ split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());
+ split.setMockInstance(mockInstance);
+ split.setFetchedColumns(tableConfig.getFetchedColumns());
+ split.setPrincipal(principal);
+ split.setToken(token);
+ split.setInstanceName(instance.getInstanceName());
+ split.setZooKeepers(instance.getZooKeepers());
+ split.setAuths(auths);
+ split.setIterators(tableConfig.getIterators());
+ split.setLogLevel(logLevel);
+
+ splits.add(split);
+ } else {
+ // don't divide ranges
+ ArrayList<String> locations = splitsToAdd.get(r);
+ if (locations == null)
+ locations = new ArrayList<String>(1);
+ locations.add(location);
+ splitsToAdd.put(r, locations);
+ }
+ }
+ }
+ }
+
+ if (!autoAdjust)
+ for (Map.Entry<Range,ArrayList<String>> entry : splitsToAdd.entrySet()) {
+ RangeInputSplit split = new RangeInputSplit(tableName, tableId, entry.getKey(), entry.getValue().toArray(new String[0]));
+
+ split.setOffline(tableConfig.isOfflineScan());
+ split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
+ split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());
+ split.setMockInstance(mockInstance);
+ split.setFetchedColumns(tableConfig.getFetchedColumns());
+ split.setPrincipal(principal);
+ split.setToken(token);
+ split.setInstanceName(instance.getInstanceName());
+ split.setZooKeepers(instance.getZooKeepers());
+ split.setAuths(auths);
+ split.setIterators(tableConfig.getIterators());
+ split.setLogLevel(logLevel);
+
+ splits.add(split);
+ }
+ }
+ return splits;
+ }
+
+ // use reflection to pull the Configuration out of the JobContext for Hadoop 1 and Hadoop 2 compatibility
+ static Configuration getConfiguration(JobContext context) {
+ try {
+ Class<?> c = AbstractInputFormat.class.getClassLoader().loadClass("org.apache.hadoop.mapreduce.JobContext");
+ Method m = c.getMethod("getConfiguration");
+ Object o = m.invoke(context, new Object[0]);
+ return (Configuration) o;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
index 0c924b1,1f83541..2c01b0d
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
@@@ -92,26 -92,7 +91,25 @@@ public class AccumuloOutputFormat exten
public static void setConnectorInfo(Job job, String principal, AuthenticationToken token) throws AccumuloSecurityException {
OutputConfigurator.setConnectorInfo(CLASS, job.getConfiguration(), principal, token);
}
-
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * Stores the password in a file in HDFS and pulls that into the Distributed Cache in an attempt to be more secure than storing it in the Configuration.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission if {@link #setCreateTables(Job, boolean)} is set to true)
+ * @param tokenFile
+ * the path to the token file
- * @throws AccumuloSecurityException
+ * @since 1.6.0
+ */
+ public static void setConnectorInfo(Job job, String principal, String tokenFile) throws AccumuloSecurityException {
+ OutputConfigurator.setConnectorInfo(CLASS, job.getConfiguration(), principal, tokenFile);
+ }
+
/**
* Determines if the connector has been configured.
*
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputTableConfig.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputTableConfig.java
index 808bd7c,0000000..e59451e
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputTableConfig.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputTableConfig.java
@@@ -1,370 -1,0 +1,367 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * This class to holds a batch scan configuration for a table. It contains all the properties needed to specify how rows should be returned from the table.
+ */
+public class InputTableConfig implements Writable {
+
+ private List<IteratorSetting> iterators;
+ private List<Range> ranges;
+ private Collection<Pair<Text,Text>> columns;
+
+ private boolean autoAdjustRanges = true;
+ private boolean useLocalIterators = false;
+ private boolean useIsolatedScanners = false;
+ private boolean offlineScan = false;
+
+ public InputTableConfig() {}
+
+ /**
+ * Creates a batch scan config object out of a previously serialized batch scan config object.
+ *
+ * @param input
+ * the data input of the serialized batch scan config
- * @throws IOException
+ */
+ public InputTableConfig(DataInput input) throws IOException {
+ readFields(input);
+ }
+
+ /**
+ * Sets the input ranges to scan for all tables associated with this job. This will be added to any per-table ranges that have been set using
+ *
+ * @param ranges
+ * the ranges that will be mapped over
+ * @since 1.6.0
+ */
+ public InputTableConfig setRanges(List<Range> ranges) {
+ this.ranges = ranges;
+ return this;
+ }
+
+ /**
+ * Returns the ranges to be queried in the configuration
+ */
+ public List<Range> getRanges() {
+ return ranges != null ? ranges : new ArrayList<Range>();
+ }
+
+ /**
+ * Restricts the columns that will be mapped over for this job for the default input table.
+ *
+ * @param columns
+ * a pair of {@link Text} objects corresponding to column family and column qualifier. If the column qualifier is null, the entire column family is
+ * selected. An empty set is the default and is equivalent to scanning the all columns.
+ * @since 1.6.0
+ */
+ public InputTableConfig fetchColumns(Collection<Pair<Text,Text>> columns) {
+ this.columns = columns;
+ return this;
+ }
+
+ /**
+ * Returns the columns to be fetched for this configuration
+ */
+ public Collection<Pair<Text,Text>> getFetchedColumns() {
+ return columns != null ? columns : new HashSet<Pair<Text,Text>>();
+ }
+
+ /**
+ * Set iterators on to be used in the query.
+ *
+ * @param iterators
+ * the configurations for the iterators
+ * @since 1.6.0
+ */
+ public InputTableConfig setIterators(List<IteratorSetting> iterators) {
+ this.iterators = iterators;
+ return this;
+ }
+
+ /**
+ * Returns the iterators to be set on this configuration
+ */
+ public List<IteratorSetting> getIterators() {
+ return iterators != null ? iterators : new ArrayList<IteratorSetting>();
+ }
+
+ /**
+ * Controls the automatic adjustment of ranges for this job. This feature merges overlapping ranges, then splits them to align with tablet boundaries.
+ * Disabling this feature will cause exactly one Map task to be created for each specified range. The default setting is enabled. *
+ *
+ * <p>
+ * By default, this feature is <b>enabled</b>.
+ *
+ * @param autoAdjustRanges
+ * the feature is enabled if true, disabled otherwise
+ * @see #setRanges(java.util.List)
+ * @since 1.6.0
+ */
+ public InputTableConfig setAutoAdjustRanges(boolean autoAdjustRanges) {
+ this.autoAdjustRanges = autoAdjustRanges;
+ return this;
+ }
+
+ /**
+ * Determines whether a configuration has auto-adjust ranges enabled.
+ *
+ * @return false if the feature is disabled, true otherwise
+ * @since 1.6.0
+ * @see #setAutoAdjustRanges(boolean)
+ */
+ public boolean shouldAutoAdjustRanges() {
+ return autoAdjustRanges;
+ }
+
+ /**
+ * Controls the use of the {@link org.apache.accumulo.core.client.ClientSideIteratorScanner} in this job. Enabling this feature will cause the iterator stack
+ * to be constructed within the Map task, rather than within the Accumulo TServer. To use this feature, all classes needed for those iterators must be
+ * available on the classpath for the task.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param useLocalIterators
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public InputTableConfig setUseLocalIterators(boolean useLocalIterators) {
+ this.useLocalIterators = useLocalIterators;
+ return this;
+ }
+
+ /**
+ * Determines whether a configuration uses local iterators.
+ *
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setUseLocalIterators(boolean)
+ */
+ public boolean shouldUseLocalIterators() {
+ return useLocalIterators;
+ }
+
+ /**
+ * <p>
+ * Enable reading offline tables. By default, this feature is disabled and only online tables are scanned. This will make the map reduce job directly read the
+ * table's files. If the table is not offline, then the job will fail. If the table comes online during the map reduce job, it is likely that the job will
+ * fail.
+ *
+ * <p>
+ * To use this option, the map reduce user will need access to read the Accumulo directory in HDFS.
+ *
+ * <p>
+ * Reading the offline table will create the scan time iterator stack in the map process. So any iterators that are configured for the table will need to be
+ * on the mapper's classpath. The accumulo-site.xml may need to be on the mapper's classpath if HDFS or the Accumulo directory in HDFS are non-standard.
+ *
+ * <p>
+ * One way to use this feature is to clone a table, take the clone offline, and use the clone as the input table for a map reduce job. If you plan to map
+ * reduce over the data many times, it may be better to the compact the table, clone it, take it offline, and use the clone for all map reduce jobs. The
+ * reason to do this is that compaction will reduce each tablet in the table to one file, and it is faster to read from one file.
+ *
+ * <p>
+ * There are two possible advantages to reading a tables file directly out of HDFS. First, you may see better read performance. Second, it will support
+ * speculative execution better. When reading an online table speculative execution can put more load on an already slow tablet server.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param offlineScan
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public InputTableConfig setOfflineScan(boolean offlineScan) {
+ this.offlineScan = offlineScan;
+ return this;
+ }
+
+ /**
+ * Determines whether a configuration has the offline table scan feature enabled.
+ *
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setOfflineScan(boolean)
+ */
+ public boolean isOfflineScan() {
+ return offlineScan;
+ }
+
+ /**
+ * Controls the use of the {@link org.apache.accumulo.core.client.IsolatedScanner} in this job.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param useIsolatedScanners
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public InputTableConfig setUseIsolatedScanners(boolean useIsolatedScanners) {
+ this.useIsolatedScanners = useIsolatedScanners;
+ return this;
+ }
+
+ /**
+ * Determines whether a configuration has isolation enabled.
+ *
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setUseIsolatedScanners(boolean)
+ */
+ public boolean shouldUseIsolatedScanners() {
+ return useIsolatedScanners;
+ }
+
+ /**
+ * Writes the state for the current object out to the specified {@link DataOutput}
+ *
+ * @param dataOutput
+ * the output for which to write the object's state
- * @throws IOException
+ */
+ @Override
+ public void write(DataOutput dataOutput) throws IOException {
+ if (iterators != null) {
+ dataOutput.writeInt(iterators.size());
+ for (IteratorSetting setting : iterators)
+ setting.write(dataOutput);
+ } else {
+ dataOutput.writeInt(0);
+ }
+ if (ranges != null) {
+ dataOutput.writeInt(ranges.size());
+ for (Range range : ranges)
+ range.write(dataOutput);
+ } else {
+ dataOutput.writeInt(0);
+ }
+ if (columns != null) {
+ dataOutput.writeInt(columns.size());
+ for (Pair<Text,Text> column : columns) {
+ if (column.getSecond() == null) {
+ dataOutput.writeInt(1);
+ column.getFirst().write(dataOutput);
+ } else {
+ dataOutput.writeInt(2);
+ column.getFirst().write(dataOutput);
+ column.getSecond().write(dataOutput);
+ }
+ }
+ } else {
+ dataOutput.writeInt(0);
+ }
+ dataOutput.writeBoolean(autoAdjustRanges);
+ dataOutput.writeBoolean(useLocalIterators);
+ dataOutput.writeBoolean(useIsolatedScanners);
+ }
+
+ /**
+ * Reads the fields in the {@link DataInput} into the current object
+ *
+ * @param dataInput
+ * the input fields to read into the current object
- * @throws IOException
+ */
+ @Override
+ public void readFields(DataInput dataInput) throws IOException {
+ // load iterators
+ long iterSize = dataInput.readInt();
+ if (iterSize > 0)
+ iterators = new ArrayList<IteratorSetting>();
+ for (int i = 0; i < iterSize; i++)
+ iterators.add(new IteratorSetting(dataInput));
+ // load ranges
+ long rangeSize = dataInput.readInt();
+ if (rangeSize > 0)
+ ranges = new ArrayList<Range>();
+ for (int i = 0; i < rangeSize; i++) {
+ Range range = new Range();
+ range.readFields(dataInput);
+ ranges.add(range);
+ }
+ // load columns
+ long columnSize = dataInput.readInt();
+ if (columnSize > 0)
+ columns = new HashSet<Pair<Text,Text>>();
+ for (int i = 0; i < columnSize; i++) {
+ long numPairs = dataInput.readInt();
+ Text colFam = new Text();
+ colFam.readFields(dataInput);
+ if (numPairs == 1) {
+ columns.add(new Pair<Text,Text>(colFam, null));
+ } else if (numPairs == 2) {
+ Text colQual = new Text();
+ colQual.readFields(dataInput);
+ columns.add(new Pair<Text,Text>(colFam, colQual));
+ }
+ }
+ autoAdjustRanges = dataInput.readBoolean();
+ useLocalIterators = dataInput.readBoolean();
+ useIsolatedScanners = dataInput.readBoolean();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ InputTableConfig that = (InputTableConfig) o;
+
+ if (autoAdjustRanges != that.autoAdjustRanges)
+ return false;
+ if (offlineScan != that.offlineScan)
+ return false;
+ if (useIsolatedScanners != that.useIsolatedScanners)
+ return false;
+ if (useLocalIterators != that.useLocalIterators)
+ return false;
+ if (columns != null ? !columns.equals(that.columns) : that.columns != null)
+ return false;
+ if (iterators != null ? !iterators.equals(that.iterators) : that.iterators != null)
+ return false;
+ if (ranges != null ? !ranges.equals(that.ranges) : that.ranges != null)
+ return false;
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = 31 * (iterators != null ? iterators.hashCode() : 0);
+ result = 31 * result + (ranges != null ? ranges.hashCode() : 0);
+ result = 31 * result + (columns != null ? columns.hashCode() : 0);
+ result = 31 * result + (autoAdjustRanges ? 1 : 0);
+ result = 31 * result + (useLocalIterators ? 1 : 0);
+ result = 31 * result + (useIsolatedScanners ? 1 : 0);
+ result = 31 * result + (offlineScan ? 1 : 0);
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
index cf861ce,1a029dc..54ff976
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
@@@ -125,43 -102,8 +124,42 @@@ public class ConfiguratorBase
ArgumentChecker.notNull(principal, token);
conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
conf.set(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL), principal);
- conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN_CLASS), token.getClass().getCanonicalName());
- conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN), CredentialHelper.tokenAsBase64(token));
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN),
+ TokenSource.INLINE.prefix() + token.getClass().getName() + ":" + Base64.encodeBase64String(AuthenticationTokenSerializer.serialize(token)));
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * Pulls a token file into the Distributed Cache that contains the authentication token in an attempt to be more secure than storing the password in the
+ * Configuration. Token file created with "bin/accumulo create-token".
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param principal
+ * a valid Accumulo user name
+ * @param tokenFile
+ * the path to the token file in DFS
- * @throws AccumuloSecurityException
+ * @since 1.6.0
+ */
+ public static void setConnectorInfo(Class<?> implementingClass, Configuration conf, String principal, String tokenFile) throws AccumuloSecurityException {
+ if (isConnectorInfoSet(implementingClass, conf))
+ throw new IllegalStateException("Connector info for " + implementingClass.getSimpleName() + " can only be set once per job");
+
+ ArgumentChecker.notNull(principal, tokenFile);
+
+ try {
+ DistributedCacheHelper.addCacheFile(new URI(tokenFile), conf);
+ } catch (URISyntaxException e) {
+ throw new IllegalStateException("Unable to add tokenFile \"" + tokenFile + "\" to distributed cache.");
+ }
+
+ conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL), principal);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN), TokenSource.FILE.prefix() + tokenFile);
}
/**
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/data/Condition.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Condition.java
index c80dcd6,0000000..bfd4818
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/data/Condition.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Condition.java
@@@ -1,245 -1,0 +1,238 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
+import java.util.Arrays;
+import java.util.HashSet;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Conditions that must be met on a particular column in a row.
+ *
+ * @since 1.6.0
+ */
+public class Condition {
+
+ private ByteSequence cf;
+ private ByteSequence cq;
+ private ByteSequence cv;
+ private ByteSequence val;
+ private Long ts;
+ private IteratorSetting iterators[] = new IteratorSetting[0];
+ private static final ByteSequence EMPTY = new ArrayByteSequence(new byte[0]);
+
+
+ public Condition(CharSequence cf, CharSequence cq) {
+ ArgumentChecker.notNull(cf, cq);
+ this.cf = new ArrayByteSequence(cf.toString().getBytes(Constants.UTF8));
+ this.cq = new ArrayByteSequence(cq.toString().getBytes(Constants.UTF8));
+ this.cv = EMPTY;
+ }
+
+ public Condition(byte[] cf, byte[] cq) {
+ ArgumentChecker.notNull(cf, cq);
+ this.cf = new ArrayByteSequence(cf);
+ this.cq = new ArrayByteSequence(cq);
+ this.cv = EMPTY;
+ }
+
+ public Condition(Text cf, Text cq) {
+ ArgumentChecker.notNull(cf, cq);
+ this.cf = new ArrayByteSequence(cf.getBytes(), 0, cf.getLength());
+ this.cq = new ArrayByteSequence(cq.getBytes(), 0, cq.getLength());
+ this.cv = EMPTY;
+ }
+
+ public Condition(ByteSequence cf, ByteSequence cq) {
+ ArgumentChecker.notNull(cf, cq);
+ this.cf = cf;
+ this.cq = cq;
+ this.cv = EMPTY;
+ }
+
+ public ByteSequence getFamily() {
+ return cf;
+ }
+
+ public ByteSequence getQualifier() {
+ return cq;
+ }
+
+ /**
+ * Sets the version for the column to check. If this is not set then the latest column will be checked, unless iterators do something different.
+ *
- * @param ts
+ * @return returns this
+ */
+
+ public Condition setTimestamp(long ts) {
+ this.ts = ts;
+ return this;
+ }
+
+ public Long getTimestamp() {
+ return ts;
+ }
+
+ /**
+ * see {@link #setValue(byte[])}
+ *
- * @param value
+ * @return returns this
+ */
+
+ public Condition setValue(CharSequence value) {
+ ArgumentChecker.notNull(value);
+ this.val = new ArrayByteSequence(value.toString().getBytes(Constants.UTF8));
+ return this;
+ }
+
+ /**
+ * This method sets the expected value of a column. Inorder for the condition to pass the column must exist and have this value. If a value is not set, then
+ * the column must be absent for the condition to pass.
+ *
- * @param value
+ * @return returns this
+ */
+
+ public Condition setValue(byte[] value) {
+ ArgumentChecker.notNull(value);
+ this.val = new ArrayByteSequence(value);
+ return this;
+ }
+
+ /**
+ * see {@link #setValue(byte[])}
+ *
- * @param value
+ * @return returns this
+ */
+
+ public Condition setValue(Text value) {
+ ArgumentChecker.notNull(value);
+ this.val = new ArrayByteSequence(value.getBytes(), 0, value.getLength());
+ return this;
+ }
+
+ /**
+ * see {@link #setValue(byte[])}
+ *
- * @param value
+ * @return returns this
+ */
+
+ public Condition setValue(ByteSequence value) {
+ ArgumentChecker.notNull(value);
+ this.val = value;
+ return this;
+ }
+
+ public ByteSequence getValue() {
+ return val;
+ }
+
+ /**
+ * Sets the visibility for the column to check. If not set it defaults to empty visibility.
+ *
- * @param cv
+ * @return returns this
+ */
+
+ public Condition setVisibility(ColumnVisibility cv) {
+ ArgumentChecker.notNull(cv);
+ this.cv = new ArrayByteSequence(cv.getExpression());
+ return this;
+ }
+
+ public ByteSequence getVisibility() {
+ return cv;
+ }
+
+ /**
+ * Set iterators to use when reading the columns value. These iterators will be applied in addition to the iterators configured for the table. Using iterators
+ * its possible to test other conditions, besides equality and absence, like less than. On the server side the iterators will be seeked using a range that
+ * covers only the family, qualifier, and visibility (if the timestamp is set then it will be used to narrow the range). Value equality will be tested using
+ * the first entry returned by the iterator stack.
+ *
- * @param iterators
+ * @return returns this
+ */
+
+ public Condition setIterators(IteratorSetting... iterators) {
+ ArgumentChecker.notNull(iterators);
+
+ if (iterators.length > 1) {
+ HashSet<String> names = new HashSet<String>();
+ HashSet<Integer> prios = new HashSet<Integer>();
+
+ for (IteratorSetting iteratorSetting : iterators) {
+ if (!names.add(iteratorSetting.getName()))
+ throw new IllegalArgumentException("iterator name used more than once " + iteratorSetting.getName());
+ if (!prios.add(iteratorSetting.getPriority()))
+ throw new IllegalArgumentException("iterator priority used more than once " + iteratorSetting.getPriority());
+ }
+ }
+
+ this.iterators = iterators;
+ return this;
+ }
+
+ public IteratorSetting[] getIterators() {
+ return iterators;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this) {
+ return true;
+ }
+ if (o == null || !(o instanceof Condition)) {
+ return false;
+ }
+ Condition c = (Condition) o;
+ if (!(c.cf.equals(cf))) {
+ return false;
+ }
+ if (!(c.cq.equals(cq))) {
+ return false;
+ }
+ if (!(c.cv.equals(cv))) {
+ return false;
+ }
+ if (!(c.val == null ? val == null : c.val.equals(val))) {
+ return false;
+ }
+ if (!(c.ts == null ? ts == null : c.ts.equals(ts))) {
+ return false;
+ }
+ if (!(Arrays.equals(c.iterators, iterators))) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = 17;
+ result = 31 * result + cf.hashCode();
+ result = 31 * result + cq.hashCode();
+ result = 31 * result + cv.hashCode();
+ result = 31 * result + (val == null ? 0 : val.hashCode());
+ result = 31 * result + (ts == null ? 0 : ts.hashCode());
+ result = 31 * result + Arrays.hashCode(iterators);
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/data/Range.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index 6c3ea0d,7d15851..ca97e01
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@@ -136,13 -113,8 +136,11 @@@ public final class BCFile
/**
* @param compressionAlgo
* The compression algorithm to be used to for compression.
+ * @param cryptoModule
+ * the module to use to obtain cryptographic streams
- * @param cryptoParams
- * @throws IOException
*/
- public WBlockState(Algorithm compressionAlgo, FSDataOutputStream fsOut, BytesWritable fsOutputBuffer, Configuration conf) throws IOException {
+ public WBlockState(Algorithm compressionAlgo, FSDataOutputStream fsOut, BytesWritable fsOutputBuffer, Configuration conf, CryptoModule cryptoModule,
+ CryptoModuleParameters cryptoParams) throws IOException {
this.compressAlgo = compressionAlgo;
this.fsOut = fsOut;
this.posStart = fsOut.getPos();
@@@ -343,10 -263,9 +339,9 @@@
* FS output stream.
* @param compressionName
* Name of the compression algorithm, which will be used for all data blocks.
- * @throws IOException
* @see Compression#getSupportedAlgorithms
*/
- public Writer(FSDataOutputStream fout, String compressionName, Configuration conf, boolean trackDataBlocks) throws IOException {
+ public Writer(FSDataOutputStream fout, String compressionName, Configuration conf, boolean trackDataBlocks, AccumuloConfiguration accumuloConfiguration) throws IOException {
if (fout.getPos() != 0) {
throw new IOException("Output file not at zero offset.");
}
@@@ -507,7 -402,8 +500,8 @@@
this.name = name;
this.compressAlgo = compressAlgo;
}
-
+
+ @Override
public void register(long raw, long begin, long end) {
metaIndex.addEntry(new MetaIndexEntry(name, compressAlgo, new BlockRegion(begin, end - begin, raw)));
}
@@@ -521,7 -417,8 +515,8 @@@
DataBlockRegister() {
// do nothing
}
-
+
+ @Override
public void register(long raw, long begin, long end) {
dataIndex.addBlockRegion(new BlockRegion(begin, end - begin, raw));
}
@@@ -734,37 -560,22 +729,36 @@@
* FS input stream.
* @param fileLength
* Length of the corresponding file
- * @throws IOException
*/
- public Reader(FSDataInputStream fin, long fileLength, Configuration conf) throws IOException {
+ public Reader(FSDataInputStream fin, long fileLength, Configuration conf, AccumuloConfiguration accumuloConfiguration) throws IOException {
+
this.in = fin;
this.conf = conf;
-
- // move the cursor to the beginning of the tail, containing: offset to the
- // meta block index, version and magic
- fin.seek(fileLength - Magic.size() - Version.size() - Long.SIZE / Byte.SIZE);
- long offsetIndexMeta = fin.readLong();
+
+ // Move the cursor to grab the version and the magic first
+ fin.seek(fileLength - Magic.size() - Version.size());
version = new Version(fin);
Magic.readAndVerify(fin);
-
- if (!version.compatibleWith(BCFile.API_VERSION)) {
+
+ // Do a version check
+ if (!version.compatibleWith(BCFile.API_VERSION) && !version.equals(BCFile.API_VERSION_1)) {
throw new RuntimeException("Incompatible BCFile fileBCFileVersion.");
}
-
+
+ // Read the right number offsets based on version
+ long offsetIndexMeta = 0;
+ long offsetCryptoParameters = 0;
+
+ if (version.equals(API_VERSION_1)) {
+ fin.seek(fileLength - Magic.size() - Version.size() - (Long.SIZE / Byte.SIZE));
+ offsetIndexMeta = fin.readLong();
+
+ } else {
+ fin.seek(fileLength - Magic.size() - Version.size() - (2 * (Long.SIZE / Byte.SIZE)));
+ offsetIndexMeta = fin.readLong();
+ offsetCryptoParameters = fin.readLong();
+ }
+
// read meta index
fin.seek(offsetIndexMeta);
metaIndex = new MetaIndex(fin);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
index 541a379,53ea0e8..0ebf4d8
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
@@@ -628,11 -628,11 +628,11 @@@ abstract public class TransformingItera
* @return the part of the key this iterator is not transforming
*/
abstract protected PartialKey getKeyPrefix();
-
- public static interface KVBuffer {
+
+ public interface KVBuffer {
void append(Key key, Value val);
}
-
+
/**
* Transforms {@code input}. This method must not change the row part of the key, and must only change the parts of the key after the return value of
* {@link #getKeyPrefix()}. Implementors must also remember to copy the delete flag from {@code originalKey} onto the new key. Or, implementors should use one
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/security/SecurityUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleFactory.java
----------------------------------------------------------------------
[54/64] [abbrv] Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
index cdcdfba,0000000..98d3c73
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
@@@ -1,276 -1,0 +1,272 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.metrics;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.lang.management.ManagementFactory;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+import javax.management.StandardMBean;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.commons.lang.builder.ToStringBuilder;
+import org.apache.commons.lang.time.DateUtils;
+
+public abstract class AbstractMetricsImpl {
+
+ public class Metric {
+
+ private long count = 0;
+ private long avg = 0;
+ private long min = 0;
+ private long max = 0;
+
+ public long getCount() {
+ return count;
+ }
+
+ public long getAvg() {
+ return avg;
+ }
+
+ public long getMin() {
+ return min;
+ }
+
+ public long getMax() {
+ return max;
+ }
+
+ public void incCount() {
+ count++;
+ }
+
+ public void addAvg(long a) {
+ if (a < 0)
+ return;
+ avg = (long) ((avg * .8) + (a * .2));
+ }
+
+ public void addMin(long a) {
+ if (a < 0)
+ return;
+ min = Math.min(min, a);
+ }
+
+ public void addMax(long a) {
+ if (a < 0)
+ return;
+ max = Math.max(max, a);
+ }
+
+ @Override
+ public String toString() {
+ return new ToStringBuilder(this).append("count", count).append("average", avg).append("minimum", min).append("maximum", max).toString();
+ }
+
+ }
+
+ static final org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger(AbstractMetricsImpl.class);
+
+ private static ConcurrentHashMap<String,Metric> registry = new ConcurrentHashMap<String,Metric>();
+
+ private boolean currentlyLogging = false;
+
+ private File logDir = null;
+
+ private String metricsPrefix = null;
+
+ private Date today = new Date();
+
+ private File logFile = null;
+
+ private Writer logWriter = null;
+
+ private SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd");
+
+ private SimpleDateFormat logFormatter = new SimpleDateFormat("yyyyMMddhhmmssz");
+
+ private MetricsConfiguration config = null;
+
+ public AbstractMetricsImpl() {
+ this.metricsPrefix = getMetricsPrefix();
+ config = new MetricsConfiguration(metricsPrefix);
+ }
+
+ /**
+ * Registers a StandardMBean with the MBean Server
- *
- * @throws Exception
+ */
+ public void register(StandardMBean mbean) throws Exception {
+ // Register this object with the MBeanServer
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ if (null == getObjectName())
+ throw new IllegalArgumentException("MBean object name must be set.");
+ mbs.registerMBean(mbean, getObjectName());
+
+ setupLogging();
+ }
+
+ /**
+ * Registers this MBean with the MBean Server
- *
- * @throws Exception
+ */
+ public void register() throws Exception {
+ // Register this object with the MBeanServer
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ if (null == getObjectName())
+ throw new IllegalArgumentException("MBean object name must be set.");
+ mbs.registerMBean(this, getObjectName());
+ setupLogging();
+ }
+
+ public void createMetric(String name) {
+ registry.put(name, new Metric());
+ }
+
+ public Metric getMetric(String name) {
+ return registry.get(name);
+ }
+
+ public long getMetricCount(String name) {
+ return registry.get(name).getCount();
+ }
+
+ public long getMetricAvg(String name) {
+ return registry.get(name).getAvg();
+ }
+
+ public long getMetricMin(String name) {
+ return registry.get(name).getMin();
+ }
+
+ public long getMetricMax(String name) {
+ return registry.get(name).getMax();
+ }
+
+ private void setupLogging() throws IOException {
+ if (null == config.getMetricsConfiguration())
+ return;
+ // If we are already logging, then return
+ if (!currentlyLogging && config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ // Check to see if directory exists, else make it
+ String mDir = config.getMetricsConfiguration().getString("logging.dir");
+ if (null != mDir) {
+ File dir = new File(mDir);
+ if (!dir.isDirectory())
+ if (!dir.mkdir())
+ log.warn("Could not create log directory: " + dir);
+ logDir = dir;
+ // Create new log file
+ startNewLog();
+ }
+ currentlyLogging = true;
+ }
+ }
+
+ private void startNewLog() throws IOException {
+ if (null != logWriter) {
+ logWriter.flush();
+ logWriter.close();
+ }
+ logFile = new File(logDir, metricsPrefix + "-" + formatter.format(today) + ".log");
+ if (!logFile.exists()) {
+ if (!logFile.createNewFile()) {
+ log.error("Unable to create new log file");
+ currentlyLogging = false;
+ return;
+ }
+ }
+ logWriter = new OutputStreamWriter(new FileOutputStream(logFile, true), Constants.UTF8);
+ }
+
+ private void writeToLog(String name) throws IOException {
+ if (null == logWriter)
+ return;
+ // Increment the date if we have to
+ Date now = new Date();
+ if (!DateUtils.isSameDay(today, now)) {
+ today = now;
+ startNewLog();
+ }
+ logWriter.append(logFormatter.format(now)).append(" Metric: ").append(name).append(": ").append(registry.get(name).toString()).append("\n");
+ }
+
+ public void add(String name, long time) {
+ if (isEnabled()) {
+ registry.get(name).incCount();
+ registry.get(name).addAvg(time);
+ registry.get(name).addMin(time);
+ registry.get(name).addMax(time);
+ // If we are not currently logging and should be, then initialize
+ if (!currentlyLogging && config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ try {
+ setupLogging();
+ } catch (IOException ioe) {
+ log.error("Error setting up log", ioe);
+ }
+ } else if (currentlyLogging && !config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ // if we are currently logging and shouldn't be, then close logs
+ try {
+ logWriter.flush();
+ logWriter.close();
+ logWriter = null;
+ logFile = null;
+ } catch (Exception e) {
+ log.error("Error stopping metrics logging", e);
+ }
+ currentlyLogging = false;
+ }
+ if (currentlyLogging) {
+ try {
+ writeToLog(name);
+ } catch (IOException ioe) {
+ log.error("Error writing to metrics log", ioe);
+ }
+ }
+ }
+ }
+
+ public boolean isEnabled() {
+ return config.isEnabled();
+ }
+
+ protected abstract ObjectName getObjectName();
+
+ protected abstract String getMetricsPrefix();
+
+ @Override
+ protected void finalize() {
+ if (null != logWriter) {
+ try {
+ logWriter.close();
+ } catch (Exception e) {
+ // do nothing
+ } finally {
+ logWriter = null;
+ }
+ }
+ logFile = null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
index 34d43f2,0000000..bbaf592
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
@@@ -1,175 -1,0 +1,171 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security.handler;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+public class ZKAuthorizor implements Authorizor {
+ private static final Logger log = Logger.getLogger(ZKAuthorizor.class);
+ private static Authorizor zkAuthorizorInstance = null;
+
+ private final String ZKUserAuths = "/Authorizations";
+
+ private String ZKUserPath;
+ private final ZooCache zooCache;
+
+ public static synchronized Authorizor getInstance() {
+ if (zkAuthorizorInstance == null)
+ zkAuthorizorInstance = new ZKAuthorizor();
+ return zkAuthorizorInstance;
+ }
+
+ public ZKAuthorizor() {
+ zooCache = new ZooCache();
+ }
+
+ @Override
+ public void initialize(String instanceId, boolean initialize) {
+ ZKUserPath = ZKSecurityTool.getInstancePath(instanceId) + "/users";
+ }
+
+ @Override
+ public Authorizations getCachedUserAuthorizations(String user) {
+ byte[] authsBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserAuths);
+ if (authsBytes != null)
+ return ZKSecurityTool.convertAuthorizations(authsBytes);
+ return Authorizations.EMPTY;
+ }
+
+ @Override
+ public boolean validSecurityHandlers(Authenticator auth, PermissionHandler pm) {
+ return true;
+ }
+
+ @Override
+ public void initializeSecurity(TCredentials itw, String rootuser) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+
+ // create the root user with all system privileges, no table privileges, and no record-level authorizations
+ Set<SystemPermission> rootPerms = new TreeSet<SystemPermission>();
+ for (SystemPermission p : SystemPermission.values())
+ rootPerms.add(p);
+ Map<String,Set<TablePermission>> tablePerms = new HashMap<String,Set<TablePermission>>();
+ // Allow the root user to flush the metadata tables
+ tablePerms.put(MetadataTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
+ tablePerms.put(RootTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
+
+ try {
+ // prep parent node of users with root username
+ if (!zoo.exists(ZKUserPath))
+ zoo.putPersistentData(ZKUserPath, rootuser.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
+
+ initUser(rootuser);
+ zoo.putPersistentData(ZKUserPath + "/" + rootuser + ZKUserAuths, ZKSecurityTool.convertAuthorizations(Authorizations.EMPTY), NodeExistsPolicy.FAIL);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
- /**
- * @param user
- * @throws AccumuloSecurityException
- */
+ @Override
+ public void initUser(String user) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ try {
+ zoo.putPersistentData(ZKUserPath + "/" + user, new byte[0], NodeExistsPolicy.SKIP);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void dropUser(String user) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserAuths, NodeMissingPolicy.SKIP);
+ zooCache.clear(ZKUserPath + "/" + user);
+ }
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ if (e.code().equals(KeeperException.Code.NONODE))
+ throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+
+ }
+ }
+
+ @Override
+ public void changeAuthorizations(String user, Authorizations authorizations) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserAuths, ZKSecurityTool.convertAuthorizations(authorizations),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean isValidAuthorizations(String user, List<ByteBuffer> auths) throws AccumuloSecurityException {
+ Collection<ByteBuffer> userauths = getCachedUserAuthorizations(user).getAuthorizationsBB();
+ for (ByteBuffer auth : auths)
+ if (!userauths.contains(auth))
+ return false;
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
index 6319653,0000000..1b7e7d3
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
@@@ -1,517 -1,0 +1,513 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security.handler;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.NamespacePermission;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.Code;
+
+/**
+ *
+ */
+public class ZKPermHandler implements PermissionHandler {
+ private static final Logger log = Logger.getLogger(ZKAuthorizor.class);
+ private static PermissionHandler zkPermHandlerInstance = null;
+
+ private String ZKUserPath;
+ private String ZKTablePath;
+ private String ZKNamespacePath;
+ private final ZooCache zooCache;
+ private final String ZKUserSysPerms = "/System";
+ private final String ZKUserTablePerms = "/Tables";
+ private final String ZKUserNamespacePerms = "/Namespaces";
+
+ public static synchronized PermissionHandler getInstance() {
+ if (zkPermHandlerInstance == null)
+ zkPermHandlerInstance = new ZKPermHandler();
+ return zkPermHandlerInstance;
+ }
+
+ @Override
+ public void initialize(String instanceId, boolean initialize) {
+ ZKUserPath = ZKSecurityTool.getInstancePath(instanceId) + "/users";
+ ZKTablePath = ZKSecurityTool.getInstancePath(instanceId) + "/tables";
+ ZKNamespacePath = ZKSecurityTool.getInstancePath(instanceId) + "/namespaces";
+ }
+
+ public ZKPermHandler() {
+ zooCache = new ZooCache();
+ }
+
+ @Override
+ public boolean hasTablePermission(String user, String table, TablePermission permission) throws TableNotFoundException {
+ byte[] serializedPerms;
+ try {
+ String path = ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table;
+ ZooReaderWriter.getRetryingInstance().sync(path);
+ serializedPerms = ZooReaderWriter.getRetryingInstance().getData(path, null);
+ } catch (KeeperException e) {
+ if (e.code() == Code.NONODE) {
+ // maybe the table was just deleted?
+ try {
+ // check for existence:
+ ZooReaderWriter.getRetryingInstance().getData(ZKTablePath + "/" + table, null);
+ // it's there, you don't have permission
+ return false;
+ } catch (InterruptedException ex) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ } catch (KeeperException ex) {
+ // not there, throw an informative exception
+ if (e.code() == Code.NONODE) {
+ throw new TableNotFoundException(null, table, "while checking permissions");
+ }
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ }
+ return false;
+ }
+ log.warn("Unhandled KeeperException, failing closed for table permission check", e);
+ return false;
+ } catch (InterruptedException e) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ }
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertTablePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean hasCachedTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertTablePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean hasNamespacePermission(String user, String namespace, NamespacePermission permission) throws NamespaceNotFoundException {
+ byte[] serializedPerms;
+ try {
+ String path = ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace;
+ ZooReaderWriter.getRetryingInstance().sync(path);
+ serializedPerms = ZooReaderWriter.getRetryingInstance().getData(path, null);
+ } catch (KeeperException e) {
+ if (e.code() == Code.NONODE) {
+ // maybe the namespace was just deleted?
+ try {
+ // check for existence:
+ ZooReaderWriter.getRetryingInstance().getData(ZKNamespacePath + "/" + namespace, null);
+ // it's there, you don't have permission
+ return false;
+ } catch (InterruptedException ex) {
+ log.warn("Unhandled InterruptedException, failing closed for namespace permission check", e);
+ return false;
+ } catch (KeeperException ex) {
+ // not there, throw an informative exception
+ if (e.code() == Code.NONODE) {
+ throw new NamespaceNotFoundException(null, namespace, "while checking permissions");
+ }
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ }
+ return false;
+ }
+ log.warn("Unhandled KeeperException, failing closed for table permission check", e);
+ return false;
+ } catch (InterruptedException e) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ }
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertNamespacePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean hasCachedNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException,
+ NamespaceNotFoundException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertNamespacePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public void grantSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ try {
+ byte[] permBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+ Set<SystemPermission> perms;
+ if (permBytes == null) {
+ perms = new TreeSet<SystemPermission>();
+ } else {
+ perms = ZKSecurityTool.convertSystemPermissions(permBytes);
+ }
+
+ if (perms.add(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(perms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void grantTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException {
+ Set<TablePermission> tablePerms;
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ if (serializedPerms != null)
+ tablePerms = ZKSecurityTool.convertTablePermissions(serializedPerms);
+ else
+ tablePerms = new TreeSet<TablePermission>();
+
+ try {
+ if (tablePerms.add(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, ZKSecurityTool.convertTablePermissions(tablePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void grantNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException {
+ Set<NamespacePermission> namespacePerms;
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
+ if (serializedPerms != null)
+ namespacePerms = ZKSecurityTool.convertNamespacePermissions(serializedPerms);
+ else
+ namespacePerms = new TreeSet<NamespacePermission>();
+
+ try {
+ if (namespacePerms.add(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, ZKSecurityTool.convertNamespacePermissions(namespacePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void revokeSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] sysPermBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+
+ // User had no system permission, nothing to revoke.
+ if (sysPermBytes == null)
+ return;
+
+ Set<SystemPermission> sysPerms = ZKSecurityTool.convertSystemPermissions(sysPermBytes);
+
+ try {
+ if (sysPerms.remove(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(sysPerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void revokeTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+
+ // User had no table permission, nothing to revoke.
+ if (serializedPerms == null)
+ return;
+
+ Set<TablePermission> tablePerms = ZKSecurityTool.convertTablePermissions(serializedPerms);
+ try {
+ if (tablePerms.remove(permission)) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ if (tablePerms.size() == 0)
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, NodeMissingPolicy.SKIP);
+ else
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, ZKSecurityTool.convertTablePermissions(tablePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void revokeNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
+
+ // User had no namespace permission, nothing to revoke.
+ if (serializedPerms == null)
+ return;
+
+ Set<NamespacePermission> namespacePerms = ZKSecurityTool.convertNamespacePermissions(serializedPerms);
+ try {
+ if (namespacePerms.remove(permission)) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ if (namespacePerms.size() == 0)
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, NodeMissingPolicy.SKIP);
+ else
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, ZKSecurityTool.convertNamespacePermissions(namespacePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void cleanTablePermissions(String table) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ for (String user : zooCache.getChildren(ZKUserPath))
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, NodeMissingPolicy.SKIP);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException("unknownUser", SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void cleanNamespacePermissions(String namespace) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ for (String user : zooCache.getChildren(ZKUserPath))
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, NodeMissingPolicy.SKIP);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException("unknownUser", SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void initializeSecurity(TCredentials itw, String rootuser) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+
+ // create the root user with all system privileges, no table privileges, and no record-level authorizations
+ Set<SystemPermission> rootPerms = new TreeSet<SystemPermission>();
+ for (SystemPermission p : SystemPermission.values())
+ rootPerms.add(p);
+ Map<String,Set<TablePermission>> tablePerms = new HashMap<String,Set<TablePermission>>();
+ // Allow the root user to flush the system tables
+ tablePerms.put(RootTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
+ tablePerms.put(MetadataTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
+ // essentially the same but on the system namespace, the ALTER_TABLE permission is now redundant
+ Map<String,Set<NamespacePermission>> namespacePerms = new HashMap<String,Set<NamespacePermission>>();
+ namespacePerms.put(Namespaces.ACCUMULO_NAMESPACE_ID, Collections.singleton(NamespacePermission.ALTER_NAMESPACE));
+ namespacePerms.put(Namespaces.ACCUMULO_NAMESPACE_ID, Collections.singleton(NamespacePermission.ALTER_TABLE));
+
+ try {
+ // prep parent node of users with root username
+ if (!zoo.exists(ZKUserPath))
+ zoo.putPersistentData(ZKUserPath, rootuser.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
+
+ initUser(rootuser);
+ zoo.putPersistentData(ZKUserPath + "/" + rootuser + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(rootPerms), NodeExistsPolicy.FAIL);
+ for (Entry<String,Set<TablePermission>> entry : tablePerms.entrySet())
+ createTablePerm(rootuser, entry.getKey(), entry.getValue());
+ for (Entry<String,Set<NamespacePermission>> entry : namespacePerms.entrySet())
+ createNamespacePerm(rootuser, entry.getKey(), entry.getValue());
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
- /**
- * @param user
- * @throws AccumuloSecurityException
- */
+ @Override
+ public void initUser(String user) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ try {
+ zoo.putPersistentData(ZKUserPath + "/" + user, new byte[0], NodeExistsPolicy.SKIP);
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms, new byte[0], NodeExistsPolicy.SKIP);
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms, new byte[0], NodeExistsPolicy.SKIP);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Sets up a new table configuration for the provided user/table. No checking for existence is done here, it should be done before calling.
+ */
+ private void createTablePerm(String user, String table, Set<TablePermission> perms) throws KeeperException, InterruptedException {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table,
+ ZKSecurityTool.convertTablePermissions(perms), NodeExistsPolicy.FAIL);
+ }
+ }
+
+ /**
+ * Sets up a new namespace configuration for the provided user/table. No checking for existence is done here, it should be done before calling.
+ */
+ private void createNamespacePerm(String user, String namespace, Set<NamespacePermission> perms) throws KeeperException, InterruptedException {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace,
+ ZKSecurityTool.convertNamespacePermissions(perms), NodeExistsPolicy.FAIL);
+ }
+ }
+
+ @Override
+ public void cleanUser(String user) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserSysPerms, NodeMissingPolicy.SKIP);
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms, NodeMissingPolicy.SKIP);
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms, NodeMissingPolicy.SKIP);
+ zooCache.clear(ZKUserPath + "/" + user);
+ }
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ if (e.code().equals(KeeperException.Code.NONODE))
+ throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+
+ }
+ }
+
+ @Override
+ public boolean hasSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] perms;
+ try {
+ String path = ZKUserPath + "/" + user + ZKUserSysPerms;
+ ZooReaderWriter.getRetryingInstance().sync(path);
+ perms = ZooReaderWriter.getRetryingInstance().getData(path, null);
+ } catch (KeeperException e) {
+ if (e.code() == Code.NONODE) {
+ return false;
+ }
+ log.warn("Unhandled KeeperException, failing closed for table permission check", e);
+ return false;
+ } catch (InterruptedException e) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ }
+
+ if (perms == null)
+ return false;
+ return ZKSecurityTool.convertSystemPermissions(perms).contains(permission);
+ }
+
+ @Override
+ public boolean hasCachedSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] perms = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+ if (perms == null)
+ return false;
+ return ZKSecurityTool.convertSystemPermissions(perms).contains(permission);
+ }
+
+ @Override
+ public boolean validSecurityHandlers(Authenticator authent, Authorizor author) {
+ return true;
+ }
+
+ @Override
+ public void initTable(String table) throws AccumuloSecurityException {
+ // All proper housekeeping is done on delete and permission granting, no work needs to be done here
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
index e16bd06,0000000..cf1a065
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
@@@ -1,62 -1,0 +1,59 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.TokenProperty;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.security.handler.Authenticator;
+import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+
+/**
+ *
+ */
+public class LoginProperties {
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ AccumuloConfiguration config = ServerConfiguration.getSystemConfiguration(HdfsZooInstance.getInstance());
+ Authenticator authenticator = AccumuloClassLoader.getClassLoader().loadClass(config.get(Property.INSTANCE_SECURITY_AUTHENTICATOR))
+ .asSubclass(Authenticator.class).newInstance();
+
+ List<Set<TokenProperty>> tokenProps = new ArrayList<Set<TokenProperty>>();
+
+ for (Class<? extends AuthenticationToken> tokenType : authenticator.getSupportedTokenTypes()) {
+ tokenProps.add(tokenType.newInstance().getProperties());
+ }
+
+ System.out.println("Supported token types for " + authenticator.getClass().getName() + " are : ");
+ for (Class<? extends AuthenticationToken> tokenType : authenticator.getSupportedTokenTypes()) {
+ System.out.println("\t" + tokenType.getName() + ", which accepts the following properties : ");
+
+ for (TokenProperty tokenProperty : tokenType.newInstance().getProperties()) {
+ System.out.println("\t\t" + tokenProperty);
+ }
+
+ System.out.println();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
index 6e5607e,0000000..37ef5f1
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
@@@ -1,128 -1,0 +1,124 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.Stack;
+
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.DefaultHandler;
+
+import com.beust.jcommander.Parameter;
+
+public class RestoreZookeeper {
+
+ private static class Restore extends DefaultHandler {
+ IZooReaderWriter zk = null;
+ Stack<String> cwd = new Stack<String>();
+ boolean overwrite = false;
+
+ Restore(IZooReaderWriter zk, boolean overwrite) {
+ this.zk = zk;
+ this.overwrite = overwrite;
+ }
+
+ @Override
+ public void startElement(String uri, String localName, String name, Attributes attributes) throws SAXException {
+ if ("node".equals(name)) {
+ String child = attributes.getValue("name");
+ if (child == null)
+ throw new RuntimeException("name attribute not set");
+ String encoding = attributes.getValue("encoding");
+ String value = attributes.getValue("value");
+ if (value == null)
+ value = "";
+ String path = cwd.lastElement() + "/" + child;
+ create(path, value, encoding);
+ cwd.push(path);
+ } else if ("dump".equals(name)) {
+ String root = attributes.getValue("root");
+ if (root.equals("/"))
+ cwd.push("");
+ else
+ cwd.push(root);
+ create(root, "", Constants.UTF8.name());
+ }
+ }
+
+ @Override
+ public void endElement(String uri, String localName, String name) throws SAXException {
+ cwd.pop();
+ }
+
+ // assume UTF-8 if not "base64"
+ private void create(String path, String value, String encoding) {
+ byte[] data = value.getBytes(Constants.UTF8);
+ if ("base64".equals(encoding))
+ data = Base64.decodeBase64(data);
+ try {
+ try {
+ zk.putPersistentData(path, data, overwrite ? NodeExistsPolicy.OVERWRITE : NodeExistsPolicy.FAIL);
+ } catch (KeeperException e) {
+ if (e.code().equals(KeeperException.Code.NODEEXISTS))
+ throw new RuntimeException(path + " exists. Remove it first.");
+ throw e;
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static class Opts extends Help {
+ @Parameter(names = {"-z", "--keepers"})
+ String keepers = "localhost:2181";
+ @Parameter(names = "--overwrite")
+ boolean overwrite = false;
+ @Parameter(names = "--file")
+ String file;
+ }
+
- /**
- * @param args
- * @throws Exception
- */
+ public static void main(String[] args) throws Exception {
+ Logger.getRootLogger().setLevel(Level.WARN);
+ Opts opts = new Opts();
+ opts.parseArgs(RestoreZookeeper.class.getName(), args);
+
+ InputStream in = System.in;
+ if (opts.file != null) {
+ in = new FileInputStream(opts.file);
+ }
+
+ SAXParserFactory factory = SAXParserFactory.newInstance();
+ SAXParser parser = factory.newSAXParser();
+ parser.parse(in, new Restore(ZooReaderWriter.getInstance(), opts.overwrite));
+ in.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
index cb932d7,0000000..9a54927
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
@@@ -1,290 -1,0 +1,287 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.NumUtil;
+import org.apache.accumulo.core.util.StringUtil;
+import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+public class TableDiskUsage {
+
+ private static final Logger log = Logger.getLogger(Logger.class);
+ private int nextInternalId = 0;
+ private Map<String,Integer> internalIds = new HashMap<String,Integer>();
+ private Map<Integer,String> externalIds = new HashMap<Integer,String>();
+ private Map<String,Integer[]> tableFiles = new HashMap<String,Integer[]>();
+ private Map<String,Long> fileSizes = new HashMap<String,Long>();
+
+ void addTable(String tableId) {
+ if (internalIds.containsKey(tableId))
+ throw new IllegalArgumentException("Already added table " + tableId);
+
+ int iid = nextInternalId++;
+
+ internalIds.put(tableId, iid);
+ externalIds.put(iid, tableId);
+ }
+
+ void linkFileAndTable(String tableId, String file) {
+ int internalId = internalIds.get(tableId);
+
+ Integer[] tables = tableFiles.get(file);
+ if (tables == null) {
+ tables = new Integer[internalIds.size()];
+ for (int i = 0; i < tables.length; i++)
+ tables[i] = 0;
+ tableFiles.put(file, tables);
+ }
+
+ tables[internalId] = 1;
+ }
+
+ void addFileSize(String file, long size) {
+ fileSizes.put(file, size);
+ }
+
+ Map<List<String>,Long> calculateUsage() {
+
+ Map<List<Integer>,Long> usage = new HashMap<List<Integer>,Long>();
+
+ for (Entry<String,Integer[]> entry : tableFiles.entrySet()) {
+ log.info("fileSizes " + fileSizes + " key " + Arrays.asList(entry.getKey()));
+ List<Integer> key = Arrays.asList(entry.getValue());
+ Long size = fileSizes.get(entry.getKey());
+
+ Long tablesUsage = usage.get(key);
+ if (tablesUsage == null)
+ tablesUsage = 0l;
+
+ tablesUsage += size;
+
+ usage.put(key, tablesUsage);
+
+ }
+
+ Map<List<String>,Long> externalUsage = new HashMap<List<String>,Long>();
+
+ for (Entry<List<Integer>,Long> entry : usage.entrySet()) {
+ List<String> externalKey = new ArrayList<String>();
+ List<Integer> key = entry.getKey();
+ for (int i = 0; i < key.size(); i++)
+ if (key.get(i) != 0)
+ externalKey.add(externalIds.get(i));
+
+ externalUsage.put(externalKey, entry.getValue());
+ }
+
+ return externalUsage;
+ }
+
+ public interface Printer {
+ void print(String line);
+ }
+
+ public static void printDiskUsage(AccumuloConfiguration acuConf, Collection<String> tables, VolumeManager fs, Connector conn, boolean humanReadable)
+ throws TableNotFoundException, IOException {
+ printDiskUsage(acuConf, tables, fs, conn, new Printer() {
+ @Override
+ public void print(String line) {
+ System.out.println(line);
+ }
+ }, humanReadable);
+ }
+
+ public static Map<TreeSet<String>,Long> getDiskUsage(AccumuloConfiguration acuConf, Set<String> tableIds, VolumeManager fs, Connector conn)
+ throws IOException {
+ TableDiskUsage tdu = new TableDiskUsage();
+
+ for (String tableId : tableIds)
+ tdu.addTable(tableId);
+
+ HashSet<String> tablesReferenced = new HashSet<String>(tableIds);
+ HashSet<String> emptyTableIds = new HashSet<String>();
+ HashSet<String> nameSpacesReferenced = new HashSet<String>();
+
+ for (String tableId : tableIds) {
+ Scanner mdScanner = null;
+ try {
+ mdScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ } catch (TableNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ mdScanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+
+ if (!mdScanner.iterator().hasNext()) {
+ emptyTableIds.add(tableId);
+ }
+
+ for (Entry<Key,Value> entry : mdScanner) {
+ String file = entry.getKey().getColumnQualifier().toString();
+ String parts[] = file.split("/");
+ String uniqueName = parts[parts.length - 1];
+ if (file.contains(":") || file.startsWith("../")) {
+ String ref = parts[parts.length - 3];
+ if (!ref.equals(tableId)) {
+ tablesReferenced.add(ref);
+ }
+ if (file.contains(":") && parts.length > 3) {
+ List<String> base = Arrays.asList(Arrays.copyOf(parts, parts.length - 3));
+ nameSpacesReferenced.add(StringUtil.join(base, "/"));
+ }
+ }
+
+ tdu.linkFileAndTable(tableId, uniqueName);
+ }
+ }
+
+ for (String tableId : tablesReferenced) {
+ for (String tableDir : nameSpacesReferenced) {
+ FileStatus[] files = fs.globStatus(new Path(tableDir + "/" + tableId + "/*/*"));
+ if (files != null) {
+ for (FileStatus fileStatus : files) {
+ // Assumes that all filenames are unique
+ String name = fileStatus.getPath().getName();
+ tdu.addFileSize(name, fileStatus.getLen());
+ }
+ }
+ }
+ }
+
+ HashMap<String,String> reverseTableIdMap = new HashMap<String,String>();
+ for (Entry<String,String> entry : conn.tableOperations().tableIdMap().entrySet())
+ reverseTableIdMap.put(entry.getValue(), entry.getKey());
+
+ TreeMap<TreeSet<String>,Long> usage = new TreeMap<TreeSet<String>,Long>(new Comparator<TreeSet<String>>() {
+
+ @Override
+ public int compare(TreeSet<String> o1, TreeSet<String> o2) {
+ int len1 = o1.size();
+ int len2 = o2.size();
+
+ int min = Math.min(len1, len2);
+
+ Iterator<String> iter1 = o1.iterator();
+ Iterator<String> iter2 = o2.iterator();
+
+ int count = 0;
+
+ while (count < min) {
+ String s1 = iter1.next();
+ String s2 = iter2.next();
+
+ int cmp = s1.compareTo(s2);
+
+ if (cmp != 0)
+ return cmp;
+
+ count++;
+ }
+
+ return len1 - len2;
+ }
+ });
+
+ for (Entry<List<String>,Long> entry : tdu.calculateUsage().entrySet()) {
+ TreeSet<String> tableNames = new TreeSet<String>();
+ for (String tableId : entry.getKey())
+ tableNames.add(reverseTableIdMap.get(tableId));
+
+ usage.put(tableNames, entry.getValue());
+ }
+
+ if (!emptyTableIds.isEmpty()) {
+ TreeSet<String> emptyTables = new TreeSet<String>();
+ for (String tableId : emptyTableIds) {
+ emptyTables.add(reverseTableIdMap.get(tableId));
+ }
+ usage.put(emptyTables, 0L);
+ }
+
+ return usage;
+ }
+
+ public static void printDiskUsage(AccumuloConfiguration acuConf, Collection<String> tables, VolumeManager fs, Connector conn, Printer printer,
+ boolean humanReadable) throws TableNotFoundException, IOException {
+
+ HashSet<String> tableIds = new HashSet<String>();
+
+ for (String tableName : tables) {
+ String tableId = conn.tableOperations().tableIdMap().get(tableName);
+ if (tableId == null)
+ throw new TableNotFoundException(null, tableName, "Table " + tableName + " not found");
+
+ tableIds.add(tableId);
+ }
+
+ Map<TreeSet<String>,Long> usage = getDiskUsage(acuConf, tableIds, fs, conn);
+
+ String valueFormat = humanReadable ? "%9s" : "%,24d";
+ for (Entry<TreeSet<String>,Long> entry : usage.entrySet()) {
+ Object value = humanReadable ? NumUtil.bigNumberForSize(entry.getValue()) : entry.getValue();
+ printer.print(String.format(valueFormat + " %s", value, entry.getKey()));
+ }
+ }
+
+ static class Opts extends ClientOpts {
+ @Parameter(description = " <table> { <table> ... } ")
+ List<String> tables = new ArrayList<String>();
+ }
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ VolumeManager fs = VolumeManagerImpl.get();
+ Opts opts = new Opts();
+ opts.parseArgs(TableDiskUsage.class.getName(), args);
+ Connector conn = opts.getConnector();
+ org.apache.accumulo.server.util.TableDiskUsage.printDiskUsage(DefaultConfiguration.getInstance(), opts.tables, fs, conn, false);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
index 2fc0bd3,0000000..34c2151
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
@@@ -1,75 -1,0 +1,72 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.util.List;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+
+import com.beust.jcommander.Parameter;
+
+public class TabletServerLocks {
+
+ static class Opts extends Help {
+ @Parameter(names="-list")
+ boolean list = false;
+ @Parameter(names="-delete")
+ String delete = null;
+ }
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+
+ Instance instance = HdfsZooInstance.getInstance();
+ String tserverPath = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
+ Opts opts = new Opts();
+ opts.parseArgs(TabletServerLocks.class.getName(), args);
+
+ ZooCache cache = new ZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
+
+ if (opts.list) {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ List<String> tabletServers = zoo.getChildren(tserverPath);
+
+ for (String tabletServer : tabletServers) {
+ byte[] lockData = ZooLock.getLockData(cache, tserverPath + "/" + tabletServer, null);
+ String holder = null;
+ if (lockData != null) {
+ holder = new String(lockData, Constants.UTF8);
+ }
+
+ System.out.printf("%32s %16s%n", tabletServer, holder);
+ }
+ } else if (opts.delete != null) {
+ ZooLock.deleteLock(tserverPath + "/" + args[1]);
+ } else {
+ System.out.println("Usage : " + TabletServerLocks.class.getName() + " -list|-delete <tserver lock>");
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/tserver/src/main/java/org/apache/accumulo/tserver/MemValue.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/MemValue.java
index 13bcdbe,0000000..f1fdde4
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/MemValue.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/MemValue.java
@@@ -1,93 -1,0 +1,95 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.accumulo.core.data.Value;
+
+/**
+ *
+ */
+public class MemValue extends Value {
+ int kvCount;
+ boolean merged = false;
+
+ /**
+ * @param value
+ * Value
+ * @param kv
+ * kv count
+ */
+ public MemValue(byte[] value, int kv) {
+ super(value);
+ this.kvCount = kv;
+ }
+
+ public MemValue() {
+ super();
+ this.kvCount = Integer.MAX_VALUE;
+ }
+
+ public MemValue(Value value, int kv) {
+ super(value);
+ this.kvCount = kv;
+ }
+
+ // Override
++ @Override
+ public void write(final DataOutput out) throws IOException {
+ if (!merged) {
+ byte[] combinedBytes = new byte[getSize() + 4];
+ System.arraycopy(value, 0, combinedBytes, 4, getSize());
+ combinedBytes[0] = (byte) (kvCount >>> 24);
+ combinedBytes[1] = (byte) (kvCount >>> 16);
+ combinedBytes[2] = (byte) (kvCount >>> 8);
+ combinedBytes[3] = (byte) (kvCount);
+ value = combinedBytes;
+ merged = true;
+ }
+ super.write(out);
+ }
+
++ @Override
+ public void set(final byte[] b) {
+ super.set(b);
+ merged = false;
+ }
+
++ @Override
+ public void copy(byte[] b) {
+ super.copy(b);
+ merged = false;
+ }
+
+ /**
+ * Takes a Value and will take out the embedded kvCount, and then return that value while replacing the Value with the original unembedded version
+ *
- * @param v
+ * @return The kvCount embedded in v.
+ */
+ public static int splitKVCount(Value v) {
+ if (v instanceof MemValue)
+ return ((MemValue) v).kvCount;
+
+ byte[] originalBytes = new byte[v.getSize() - 4];
+ byte[] combined = v.get();
+ System.arraycopy(combined, 4, originalBytes, 0, originalBytes.length);
+ v.set(originalBytes);
+ return (combined[0] << 24) + ((combined[1] & 0xFF) << 16) + ((combined[2] & 0xFF) << 8) + (combined[3] & 0xFF);
+ }
+}
[04/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
index ad3d615,0000000..b9b68bb
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
@@@ -1,3620 -1,0 +1,3614 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.tabletserver;
+
+import static org.apache.accumulo.server.problems.ProblemType.TABLET_LOAD;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.lang.reflect.Field;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TimerTask;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.UUID;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.RunnableFuture;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.management.ObjectName;
+import javax.management.StandardMBean;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
+import org.apache.accumulo.core.client.impl.TabletType;
+import org.apache.accumulo.core.client.impl.Translator;
++import org.apache.accumulo.core.client.impl.Translators;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
- import org.apache.accumulo.core.client.impl.Translators;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.constraints.Constraint.Environment;
+import org.apache.accumulo.core.constraints.Violations;
+import org.apache.accumulo.core.data.Column;
+import org.apache.accumulo.core.data.ConstraintViolationSummary;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.thrift.InitialMultiScan;
+import org.apache.accumulo.core.data.thrift.InitialScan;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.data.thrift.MapFileInfo;
+import org.apache.accumulo.core.data.thrift.MultiScanResult;
+import org.apache.accumulo.core.data.thrift.ScanResult;
+import org.apache.accumulo.core.data.thrift.TColumn;
+import org.apache.accumulo.core.data.thrift.TKey;
+import org.apache.accumulo.core.data.thrift.TKeyExtent;
+import org.apache.accumulo.core.data.thrift.TKeyValue;
+import org.apache.accumulo.core.data.thrift.TMutation;
+import org.apache.accumulo.core.data.thrift.TRange;
+import org.apache.accumulo.core.data.thrift.UpdateErrors;
+import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.iterators.IterationInterruptedException;
+import org.apache.accumulo.core.master.thrift.Compacting;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.master.thrift.TableInfo;
+import org.apache.accumulo.core.master.thrift.TabletLoadState;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
+import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
+import org.apache.accumulo.core.tabletserver.thrift.NoSuchScanIDException;
+import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
+import org.apache.accumulo.core.tabletserver.thrift.ScanState;
+import org.apache.accumulo.core.tabletserver.thrift.ScanType;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Processor;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.LoggingRunnable;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.ServerServices;
+import org.apache.accumulo.core.util.ServerServices.Service;
+import org.apache.accumulo.core.util.SimpleThreadPool;
+import org.apache.accumulo.core.util.Stat;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockWatcher;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.client.ClientServiceHandler;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.conf.TableConfiguration;
+import org.apache.accumulo.server.data.ServerMutation;
+import org.apache.accumulo.server.logger.LogFileKey;
+import org.apache.accumulo.server.logger.LogFileValue;
+import org.apache.accumulo.server.master.state.Assignment;
+import org.apache.accumulo.server.master.state.DistributedStoreException;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletLocationState;
+import org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException;
+import org.apache.accumulo.server.master.state.TabletStateStore;
+import org.apache.accumulo.server.master.state.ZooTabletStateStore;
+import org.apache.accumulo.server.metrics.AbstractMetricsImpl;
+import org.apache.accumulo.server.problems.ProblemReport;
+import org.apache.accumulo.server.problems.ProblemReports;
+import org.apache.accumulo.server.security.AuditedSecurityOperation;
+import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.accumulo.server.security.SecurityOperation;
+import org.apache.accumulo.server.tabletserver.Compactor.CompactionInfo;
+import org.apache.accumulo.server.tabletserver.Tablet.CommitSession;
+import org.apache.accumulo.server.tabletserver.Tablet.KVEntry;
+import org.apache.accumulo.server.tabletserver.Tablet.LookupResult;
+import org.apache.accumulo.server.tabletserver.Tablet.MajorCompactionReason;
+import org.apache.accumulo.server.tabletserver.Tablet.MinorCompactionReason;
+import org.apache.accumulo.server.tabletserver.Tablet.ScanBatch;
+import org.apache.accumulo.server.tabletserver.Tablet.Scanner;
+import org.apache.accumulo.server.tabletserver.Tablet.SplitInfo;
+import org.apache.accumulo.server.tabletserver.Tablet.TConstraintViolationException;
+import org.apache.accumulo.server.tabletserver.Tablet.TabletClosedException;
+import org.apache.accumulo.server.tabletserver.TabletServerResourceManager.TabletResourceManager;
+import org.apache.accumulo.server.tabletserver.TabletStatsKeeper.Operation;
+import org.apache.accumulo.server.tabletserver.log.DfsLogger;
+import org.apache.accumulo.server.tabletserver.log.LogSorter;
+import org.apache.accumulo.server.tabletserver.log.MutationReceiver;
+import org.apache.accumulo.server.tabletserver.log.TabletServerLogger;
+import org.apache.accumulo.server.tabletserver.mastermessage.MasterMessage;
+import org.apache.accumulo.server.tabletserver.mastermessage.SplitReportMessage;
+import org.apache.accumulo.server.tabletserver.mastermessage.TabletStatusMessage;
+import org.apache.accumulo.server.tabletserver.metrics.TabletServerMBean;
+import org.apache.accumulo.server.tabletserver.metrics.TabletServerMinCMetrics;
+import org.apache.accumulo.server.tabletserver.metrics.TabletServerScanMetrics;
+import org.apache.accumulo.server.tabletserver.metrics.TabletServerUpdateMetrics;
+import org.apache.accumulo.server.trace.TraceFileSystem;
+import org.apache.accumulo.server.util.FileSystemMonitor;
+import org.apache.accumulo.server.util.Halt;
+import org.apache.accumulo.server.util.MapCounter;
+import org.apache.accumulo.server.util.MetadataTable;
+import org.apache.accumulo.server.util.MetadataTable.LogEntry;
+import org.apache.accumulo.server.util.TServerUtils;
+import org.apache.accumulo.server.util.TServerUtils.ServerPort;
+import org.apache.accumulo.server.util.time.RelativeTime;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
+import org.apache.accumulo.server.zookeeper.TransactionWatcher;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.start.Platform;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.accumulo.start.classloader.vfs.ContextManager;
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.instrument.Trace;
+import org.apache.accumulo.trace.instrument.thrift.TraceWrap;
+import org.apache.accumulo.trace.thrift.TInfo;
+import org.apache.commons.collections.map.LRUMap;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.Trash;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.SequenceFile.Reader;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+import org.apache.thrift.TProcessor;
+import org.apache.thrift.TServiceClient;
+import org.apache.thrift.server.TServer;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+
+enum ScanRunState {
+ QUEUED, RUNNING, FINISHED
+}
+
+public class TabletServer extends AbstractMetricsImpl implements org.apache.accumulo.server.tabletserver.metrics.TabletServerMBean {
+ private static final Logger log = Logger.getLogger(TabletServer.class);
+
+ private static HashMap<String,Long> prevGcTime = new HashMap<String,Long>();
+ private static long lastMemorySize = 0;
+ private static long gcTimeIncreasedCount;
+
+ private static final long MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS = 1000;
+
+ private TabletServerLogger logger;
+
+ protected TabletServerMinCMetrics mincMetrics = new TabletServerMinCMetrics();
+
+ private ServerConfiguration serverConfig;
+ private LogSorter logSorter = null;
+
+ public TabletServer(ServerConfiguration conf, FileSystem fs) {
+ super();
+ this.serverConfig = conf;
+ this.instance = conf.getInstance();
+ this.fs = TraceFileSystem.wrap(fs);
+ this.logSorter = new LogSorter(instance, fs, getSystemConfiguration());
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ synchronized (onlineTablets) {
+ long now = System.currentTimeMillis();
+ for (Tablet tablet : onlineTablets.values())
+ try {
+ tablet.updateRates(now);
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ }
+ }
+ }, 5000, 5000);
+ }
+
+ private synchronized static void logGCInfo(AccumuloConfiguration conf) {
+ List<GarbageCollectorMXBean> gcmBeans = ManagementFactory.getGarbageCollectorMXBeans();
+ Runtime rt = Runtime.getRuntime();
+
+ StringBuilder sb = new StringBuilder("gc");
+
+ boolean sawChange = false;
+
+ long maxIncreaseInCollectionTime = 0;
+
+ for (GarbageCollectorMXBean gcBean : gcmBeans) {
+ Long prevTime = prevGcTime.get(gcBean.getName());
+ long pt = 0;
+ if (prevTime != null) {
+ pt = prevTime;
+ }
+
+ long time = gcBean.getCollectionTime();
+
+ if (time - pt != 0) {
+ sawChange = true;
+ }
+
+ long increaseInCollectionTime = time - pt;
+ sb.append(String.format(" %s=%,.2f(+%,.2f) secs", gcBean.getName(), time / 1000.0, increaseInCollectionTime / 1000.0));
+ maxIncreaseInCollectionTime = Math.max(increaseInCollectionTime, maxIncreaseInCollectionTime);
+ prevGcTime.put(gcBean.getName(), time);
+ }
+
+ long mem = rt.freeMemory();
+ if (maxIncreaseInCollectionTime == 0) {
+ gcTimeIncreasedCount = 0;
+ } else {
+ gcTimeIncreasedCount++;
+ if (gcTimeIncreasedCount > 3 && mem < rt.maxMemory() * 0.05) {
+ log.warn("Running low on memory");
+ gcTimeIncreasedCount = 0;
+ }
+ }
+
+ if (mem > lastMemorySize) {
+ sawChange = true;
+ }
+
+ String sign = "+";
+ if (mem - lastMemorySize <= 0) {
+ sign = "";
+ }
+
+ sb.append(String.format(" freemem=%,d(%s%,d) totalmem=%,d", mem, sign, (mem - lastMemorySize), rt.totalMemory()));
+
+ if (sawChange) {
+ log.debug(sb.toString());
+ }
+
+ final long keepAliveTimeout = conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT);
+ if (maxIncreaseInCollectionTime > keepAliveTimeout) {
+ Halt.halt("Garbage collection may be interfering with lock keep-alive. Halting.", -1);
+ }
+
+ lastMemorySize = mem;
+ }
+
+ private TabletStatsKeeper statsKeeper;
+
+ private static class Session {
+ long lastAccessTime;
+ long startTime;
+ String user;
+ String client = TServerUtils.clientAddress.get();
+ public boolean reserved;
+
+ public void cleanup() {}
+ }
+
+ private static class SessionManager {
+
+ SecureRandom random;
+ Map<Long,Session> sessions;
+
+ SessionManager(AccumuloConfiguration conf) {
+ random = new SecureRandom();
+ sessions = new HashMap<Long,Session>();
+
+ final long maxIdle = conf.getTimeInMillis(Property.TSERV_SESSION_MAXIDLE);
+
+ Runnable r = new Runnable() {
+ @Override
+ public void run() {
+ sweep(maxIdle);
+ }
+ };
+
+ SimpleTimer.getInstance().schedule(r, 0, Math.max(maxIdle / 2, 1000));
+ }
+
+ synchronized long createSession(Session session, boolean reserve) {
+ long sid = random.nextLong();
+
+ while (sessions.containsKey(sid)) {
+ sid = random.nextLong();
+ }
+
+ sessions.put(sid, session);
+
+ session.reserved = reserve;
+
+ session.startTime = session.lastAccessTime = System.currentTimeMillis();
+
+ return sid;
+ }
+
+ /**
+ * while a session is reserved, it cannot be canceled or removed
+ *
+ * @param sessionId
+ */
+
+ synchronized Session reserveSession(long sessionId) {
+ Session session = sessions.get(sessionId);
+ if (session != null) {
+ if (session.reserved)
+ throw new IllegalStateException();
+ session.reserved = true;
+ }
+
+ return session;
+
+ }
+
+ synchronized void unreserveSession(Session session) {
+ if (!session.reserved)
+ throw new IllegalStateException();
+ session.reserved = false;
+ session.lastAccessTime = System.currentTimeMillis();
+ }
+
+ synchronized void unreserveSession(long sessionId) {
+ Session session = getSession(sessionId);
+ if (session != null)
+ unreserveSession(session);
+ }
+
+ synchronized Session getSession(long sessionId) {
+ Session session = sessions.get(sessionId);
+ if (session != null)
+ session.lastAccessTime = System.currentTimeMillis();
+ return session;
+ }
+
+ Session removeSession(long sessionId) {
+ Session session = null;
+ synchronized (this) {
+ session = sessions.remove(sessionId);
+ }
+
+ // do clean up out side of lock..
+ if (session != null)
+ session.cleanup();
+
+ return session;
+ }
+
+ private void sweep(long maxIdle) {
+ ArrayList<Session> sessionsToCleanup = new ArrayList<Session>();
+ synchronized (this) {
+ Iterator<Session> iter = sessions.values().iterator();
+ while (iter.hasNext()) {
+ Session session = iter.next();
+ long idleTime = System.currentTimeMillis() - session.lastAccessTime;
+ if (idleTime > maxIdle && !session.reserved) {
+ iter.remove();
+ sessionsToCleanup.add(session);
+ }
+ }
+ }
+
+ // do clean up outside of lock
+ for (Session session : sessionsToCleanup) {
+ session.cleanup();
+ }
+ }
+
+ synchronized void removeIfNotAccessed(final long sessionId, long delay) {
+ Session session = sessions.get(sessionId);
+ if (session != null) {
+ final long removeTime = session.lastAccessTime;
+ TimerTask r = new TimerTask() {
+ @Override
+ public void run() {
+ Session sessionToCleanup = null;
+ synchronized (SessionManager.this) {
+ Session session2 = sessions.get(sessionId);
+ if (session2 != null && session2.lastAccessTime == removeTime && !session2.reserved) {
+ sessions.remove(sessionId);
+ sessionToCleanup = session2;
+ }
+ }
+
+ // call clean up outside of lock
+ if (sessionToCleanup != null)
+ sessionToCleanup.cleanup();
+ }
+ };
+
+ SimpleTimer.getInstance().schedule(r, delay);
+ }
+ }
+
+ public synchronized Map<String,MapCounter<ScanRunState>> getActiveScansPerTable() {
+ Map<String,MapCounter<ScanRunState>> counts = new HashMap<String,MapCounter<ScanRunState>>();
+ for (Entry<Long,Session> entry : sessions.entrySet()) {
+
+ Session session = entry.getValue();
+ @SuppressWarnings("rawtypes")
+ ScanTask nbt = null;
+ String tableID = null;
+
+ if (session instanceof ScanSession) {
+ ScanSession ss = (ScanSession) session;
+ nbt = ss.nextBatchTask;
+ tableID = ss.extent.getTableId().toString();
+ } else if (session instanceof MultiScanSession) {
+ MultiScanSession mss = (MultiScanSession) session;
+ nbt = mss.lookupTask;
+ tableID = mss.threadPoolExtent.getTableId().toString();
+ }
+
+ if (nbt == null)
+ continue;
+
+ ScanRunState srs = nbt.getScanRunState();
+
+ if (srs == ScanRunState.FINISHED)
+ continue;
+
+ MapCounter<ScanRunState> stateCounts = counts.get(tableID);
+ if (stateCounts == null) {
+ stateCounts = new MapCounter<ScanRunState>();
+ counts.put(tableID, stateCounts);
+ }
+
+ stateCounts.increment(srs, 1);
+ }
+
+ return counts;
+ }
+
+ public synchronized List<ActiveScan> getActiveScans() {
+
+ ArrayList<ActiveScan> activeScans = new ArrayList<ActiveScan>();
+
+ long ct = System.currentTimeMillis();
+
+ for (Entry<Long,Session> entry : sessions.entrySet()) {
+ Session session = entry.getValue();
+ if (session instanceof ScanSession) {
+ ScanSession ss = (ScanSession) session;
+
+ ScanState state = ScanState.RUNNING;
+
+ ScanTask<ScanBatch> nbt = ss.nextBatchTask;
+ if (nbt == null) {
+ state = ScanState.IDLE;
+ } else {
+ switch (nbt.getScanRunState()) {
+ case QUEUED:
+ state = ScanState.QUEUED;
+ break;
+ case FINISHED:
+ state = ScanState.IDLE;
+ break;
+ case RUNNING:
+ default:
+ /* do nothing */
+ break;
+ }
+ }
+
+ activeScans.add(new ActiveScan(ss.client, ss.user, ss.extent.getTableId().toString(), ct - ss.startTime, ct - ss.lastAccessTime, ScanType.SINGLE,
+ state, ss.extent.toThrift(), Translator.translate(ss.columnSet, Translators.CT), ss.ssiList, ss.ssio, ss.auths.getAuthorizationsBB()));
+
+ } else if (session instanceof MultiScanSession) {
+ MultiScanSession mss = (MultiScanSession) session;
+
+ ScanState state = ScanState.RUNNING;
+
+ ScanTask<MultiScanResult> nbt = mss.lookupTask;
+ if (nbt == null) {
+ state = ScanState.IDLE;
+ } else {
+ switch (nbt.getScanRunState()) {
+ case QUEUED:
+ state = ScanState.QUEUED;
+ break;
+ case FINISHED:
+ state = ScanState.IDLE;
+ break;
+ case RUNNING:
+ default:
+ /* do nothing */
+ break;
+ }
+ }
+
+ activeScans.add(new ActiveScan(mss.client, mss.user, mss.threadPoolExtent.getTableId().toString(), ct - mss.startTime, ct - mss.lastAccessTime,
+ ScanType.BATCH, state, mss.threadPoolExtent.toThrift(), Translator.translate(mss.columnSet, Translators.CT), mss.ssiList, mss.ssio, mss.auths
+ .getAuthorizationsBB()));
+ }
+ }
+
+ return activeScans;
+ }
+ }
+
+ static class TservConstraintEnv implements Environment {
+
+ private TCredentials credentials;
+ private SecurityOperation security;
+ private Authorizations auths;
+ private KeyExtent ke;
+
+ TservConstraintEnv(SecurityOperation secOp, TCredentials credentials) {
+ this.security = secOp;
+ this.credentials = credentials;
+ }
+
+ void setExtent(KeyExtent ke) {
+ this.ke = ke;
+ }
+
+ @Override
+ public KeyExtent getExtent() {
+ return ke;
+ }
+
+ @Override
+ public String getUser() {
+ return credentials.getPrincipal();
+ }
+
+ @Override
+ public Authorizations getAuthorizations() {
+ if (auths == null)
+ try {
+ this.auths = security.getUserAuthorizations(credentials);
+ } catch (ThriftSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ return auths;
+ }
+
+ }
+
+ private abstract class ScanTask<T> implements RunnableFuture<T> {
+
+ protected AtomicBoolean interruptFlag;
+ protected ArrayBlockingQueue<Object> resultQueue;
+ protected AtomicInteger state;
+ protected AtomicReference<ScanRunState> runState;
+
+ private static final int INITIAL = 1;
+ private static final int ADDED = 2;
+ private static final int CANCELED = 3;
+
+ ScanTask() {
+ interruptFlag = new AtomicBoolean(false);
+ runState = new AtomicReference<ScanRunState>(ScanRunState.QUEUED);
+ state = new AtomicInteger(INITIAL);
+ resultQueue = new ArrayBlockingQueue<Object>(1);
+ }
+
+ protected void addResult(Object o) {
+ if (state.compareAndSet(INITIAL, ADDED))
+ resultQueue.add(o);
+ else if (state.get() == ADDED)
+ throw new IllegalStateException("Tried to add more than one result");
+ }
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ if (!mayInterruptIfRunning)
+ throw new IllegalArgumentException("Cancel will always attempt to interupt running next batch task");
+
+ if (state.get() == CANCELED)
+ return true;
+
+ if (state.compareAndSet(INITIAL, CANCELED)) {
+ interruptFlag.set(true);
+ resultQueue = null;
+ return true;
+ }
+
+ return false;
+ }
+
+ @Override
+ public T get() throws InterruptedException, ExecutionException {
+ throw new UnsupportedOperationException();
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
+
+ ArrayBlockingQueue<Object> localRQ = resultQueue;
+
+ if (state.get() == CANCELED)
+ throw new CancellationException();
+
+ if (localRQ == null && state.get() == ADDED)
+ throw new IllegalStateException("Tried to get result twice");
+
+ Object r = localRQ.poll(timeout, unit);
+
+ // could have been canceled while waiting
+ if (state.get() == CANCELED) {
+ if (r != null)
+ throw new IllegalStateException("Nothing should have been added when in canceled state");
+
+ throw new CancellationException();
+ }
+
+ if (r == null)
+ throw new TimeoutException();
+
+ // make this method stop working now that something is being
+ // returned
+ resultQueue = null;
+
+ if (r instanceof Throwable)
+ throw new ExecutionException((Throwable) r);
+
+ return (T) r;
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return state.get() == CANCELED;
+ }
+
+ @Override
+ public boolean isDone() {
+ return runState.get().equals(ScanRunState.FINISHED);
+ }
+
+ public ScanRunState getScanRunState() {
+ return runState.get();
+ }
+
+ }
+
+ private static class UpdateSession extends Session {
+ public Tablet currentTablet;
+ public MapCounter<Tablet> successfulCommits = new MapCounter<Tablet>();
+ Map<KeyExtent,Long> failures = new HashMap<KeyExtent,Long>();
+ HashMap<KeyExtent,SecurityErrorCode> authFailures = new HashMap<KeyExtent,SecurityErrorCode>();
+ public Violations violations;
+ public TCredentials credentials;
+ public long totalUpdates = 0;
+ public long flushTime = 0;
+ Stat prepareTimes = new Stat();
+ Stat walogTimes = new Stat();
+ Stat commitTimes = new Stat();
+ Stat authTimes = new Stat();
+ public Map<Tablet,List<Mutation>> queuedMutations = new HashMap<Tablet,List<Mutation>>();
+ public long queuedMutationSize = 0;
+ TservConstraintEnv cenv = null;
+ }
+
+ private static class ScanSession extends Session {
+ public KeyExtent extent;
+ public HashSet<Column> columnSet;
+ public List<IterInfo> ssiList;
+ public Map<String,Map<String,String>> ssio;
+ public Authorizations auths;
+ public long entriesReturned = 0;
+ public Stat nbTimes = new Stat();
+ public long batchCount = 0;
+ public volatile ScanTask<ScanBatch> nextBatchTask;
+ public AtomicBoolean interruptFlag;
+ public Scanner scanner;
+
+ @Override
+ public void cleanup() {
+ try {
+ if (nextBatchTask != null)
+ nextBatchTask.cancel(true);
+ } finally {
+ if (scanner != null)
+ scanner.close();
+ }
+ }
+
+ }
+
+ private static class MultiScanSession extends Session {
+ HashSet<Column> columnSet;
+ Map<KeyExtent,List<Range>> queries;
+ public List<IterInfo> ssiList;
+ public Map<String,Map<String,String>> ssio;
+ public Authorizations auths;
+
+ // stats
+ int numRanges;
+ int numTablets;
+ int numEntries;
+ long totalLookupTime;
+
+ public volatile ScanTask<MultiScanResult> lookupTask;
+ public KeyExtent threadPoolExtent;
+
+ @Override
+ public void cleanup() {
+ if (lookupTask != null)
+ lookupTask.cancel(true);
+ }
+ }
+
+ /**
+ * This little class keeps track of writes in progress and allows readers to wait for writes that started before the read. It assumes that the operation ids
+ * are monotonically increasing.
+ *
+ */
+ static class WriteTracker {
+ private static AtomicLong operationCounter = new AtomicLong(1);
+ private Map<TabletType,TreeSet<Long>> inProgressWrites = new EnumMap<TabletType,TreeSet<Long>>(TabletType.class);
+
+ WriteTracker() {
+ for (TabletType ttype : TabletType.values()) {
+ inProgressWrites.put(ttype, new TreeSet<Long>());
+ }
+ }
+
+ synchronized long startWrite(TabletType ttype) {
+ long operationId = operationCounter.getAndIncrement();
+ inProgressWrites.get(ttype).add(operationId);
+ return operationId;
+ }
+
+ synchronized void finishWrite(long operationId) {
+ if (operationId == -1)
+ return;
+
+ boolean removed = false;
+
+ for (TabletType ttype : TabletType.values()) {
+ removed = inProgressWrites.get(ttype).remove(operationId);
+ if (removed)
+ break;
+ }
+
+ if (!removed) {
+ throw new IllegalArgumentException("Attempted to finish write not in progress, operationId " + operationId);
+ }
+
+ this.notifyAll();
+ }
+
+ synchronized void waitForWrites(TabletType ttype) {
+ long operationId = operationCounter.getAndIncrement();
+ while (inProgressWrites.get(ttype).floor(operationId) != null) {
+ try {
+ this.wait();
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ }
+ }
+ }
+
+ public long startWrite(Set<Tablet> keySet) {
+ if (keySet.size() == 0)
+ return -1;
+
+ ArrayList<KeyExtent> extents = new ArrayList<KeyExtent>(keySet.size());
+
+ for (Tablet tablet : keySet)
+ extents.add(tablet.getExtent());
+
+ return startWrite(TabletType.type(extents));
+ }
+ }
+
+ public AccumuloConfiguration getSystemConfiguration() {
+ return serverConfig.getConfiguration();
+ }
+
+ TransactionWatcher watcher = new TransactionWatcher();
+
+ private class ThriftClientHandler extends ClientServiceHandler implements TabletClientService.Iface {
+
+ SessionManager sessionManager;
+
+ AccumuloConfiguration acuConf = getSystemConfiguration();
+
+ TabletServerUpdateMetrics updateMetrics = new TabletServerUpdateMetrics();
+
+ TabletServerScanMetrics scanMetrics = new TabletServerScanMetrics();
+
+ WriteTracker writeTracker = new WriteTracker();
+
+ ThriftClientHandler() {
+ super(instance, watcher);
+ log.debug(ThriftClientHandler.class.getName() + " created");
+ sessionManager = new SessionManager(getSystemConfiguration());
+ // Register the metrics MBean
+ try {
+ updateMetrics.register();
+ scanMetrics.register();
+ } catch (Exception e) {
+ log.error("Exception registering MBean with MBean Server", e);
+ }
+ }
+
+ @Override
+ public List<TKeyExtent> bulkImport(TInfo tinfo, TCredentials credentials, long tid, Map<TKeyExtent,Map<String,MapFileInfo>> files, boolean setTime)
+ throws ThriftSecurityException {
+
+ if (!security.canPerformSystemActions(credentials))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ List<TKeyExtent> failures = new ArrayList<TKeyExtent>();
+
+ for (Entry<TKeyExtent,Map<String,MapFileInfo>> entry : files.entrySet()) {
+ TKeyExtent tke = entry.getKey();
+ Map<String,MapFileInfo> fileMap = entry.getValue();
+
+ Tablet importTablet = onlineTablets.get(new KeyExtent(tke));
+
+ if (importTablet == null) {
+ failures.add(tke);
+ } else {
+ try {
+ importTablet.importMapFiles(tid, fileMap, setTime);
+ } catch (IOException ioe) {
+ log.info("files " + fileMap.keySet() + " not imported to " + new KeyExtent(tke) + ": " + ioe.getMessage());
+ failures.add(tke);
+ }
+ }
+ }
+ return failures;
+ }
+
+ private class NextBatchTask extends ScanTask<ScanBatch> {
+
+ private long scanID;
+
+ NextBatchTask(long scanID, AtomicBoolean interruptFlag) {
+ this.scanID = scanID;
+ this.interruptFlag = interruptFlag;
+
+ if (interruptFlag.get())
+ cancel(true);
+ }
+
+ @Override
+ public void run() {
+
+ final ScanSession scanSession = (ScanSession) sessionManager.getSession(scanID);
+ String oldThreadName = Thread.currentThread().getName();
+
+ try {
+ if (isCancelled() || scanSession == null)
+ return;
+
+ runState.set(ScanRunState.RUNNING);
+
+ Thread.currentThread().setName(
+ "User: " + scanSession.user + " Start: " + scanSession.startTime + " Client: " + scanSession.client + " Tablet: " + scanSession.extent);
+
+ Tablet tablet = onlineTablets.get(scanSession.extent);
+
+ if (tablet == null) {
+ addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
+ return;
+ }
+
+ long t1 = System.currentTimeMillis();
+ ScanBatch batch = scanSession.scanner.read();
+ long t2 = System.currentTimeMillis();
+ scanSession.nbTimes.addStat(t2 - t1);
+
+ // there should only be one thing on the queue at a time, so
+ // it should be ok to call add()
+ // instead of put()... if add() fails because queue is at
+ // capacity it means there is code
+ // problem somewhere
+ addResult(batch);
+ } catch (TabletClosedException e) {
+ addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
+ } catch (IterationInterruptedException iie) {
+ if (!isCancelled()) {
+ log.warn("Iteration interrupted, when scan not cancelled", iie);
+ addResult(iie);
+ }
+ } catch (TooManyFilesException tmfe) {
+ addResult(tmfe);
+ } catch (Throwable e) {
+ log.warn("exception while scanning tablet " + (scanSession == null ? "(unknown)" : scanSession.extent), e);
+ addResult(e);
+ } finally {
+ runState.set(ScanRunState.FINISHED);
+ Thread.currentThread().setName(oldThreadName);
+ }
+
+ }
+ }
+
+ private class LookupTask extends ScanTask<MultiScanResult> {
+
+ private long scanID;
+
+ LookupTask(long scanID) {
+ this.scanID = scanID;
+ }
+
+ @Override
+ public void run() {
+ MultiScanSession session = (MultiScanSession) sessionManager.getSession(scanID);
+ String oldThreadName = Thread.currentThread().getName();
+
+ try {
+ if (isCancelled() || session == null)
+ return;
+
+ TableConfiguration acuTableConf = ServerConfiguration.getTableConfiguration(instance, session.threadPoolExtent.getTableId().toString());
+ long maxResultsSize = acuTableConf.getMemoryInBytes(Property.TABLE_SCAN_MAXMEM);
+
+ runState.set(ScanRunState.RUNNING);
+ Thread.currentThread().setName("Client: " + session.client + " User: " + session.user + " Start: " + session.startTime + " Table: ");
+
+ long bytesAdded = 0;
+ long maxScanTime = 4000;
+
+ long startTime = System.currentTimeMillis();
+
+ ArrayList<KVEntry> results = new ArrayList<KVEntry>();
+ Map<KeyExtent,List<Range>> failures = new HashMap<KeyExtent,List<Range>>();
+ ArrayList<KeyExtent> fullScans = new ArrayList<KeyExtent>();
+ KeyExtent partScan = null;
+ Key partNextKey = null;
+ boolean partNextKeyInclusive = false;
+
+ Iterator<Entry<KeyExtent,List<Range>>> iter = session.queries.entrySet().iterator();
+
+ // check the time so that the read ahead thread is not monopolized
+ while (iter.hasNext() && bytesAdded < maxResultsSize && (System.currentTimeMillis() - startTime) < maxScanTime) {
+ Entry<KeyExtent,List<Range>> entry = iter.next();
+
+ iter.remove();
+
+ // check that tablet server is serving requested tablet
+ Tablet tablet = onlineTablets.get(entry.getKey());
+ if (tablet == null) {
+ failures.put(entry.getKey(), entry.getValue());
+ continue;
+ }
+ Thread.currentThread().setName(
+ "Client: " + session.client + " User: " + session.user + " Start: " + session.startTime + " Tablet: " + entry.getKey().toString());
+
+ LookupResult lookupResult;
+ try {
+
+ // do the following check to avoid a race condition
+ // between setting false below and the task being
+ // canceled
+ if (isCancelled())
+ interruptFlag.set(true);
+
+ lookupResult = tablet.lookup(entry.getValue(), session.columnSet, session.auths, results, maxResultsSize - bytesAdded, session.ssiList,
+ session.ssio, interruptFlag);
+
+ // if the tablet was closed it it possible that the
+ // interrupt flag was set.... do not want it set for
+ // the next
+ // lookup
+ interruptFlag.set(false);
+
+ } catch (IOException e) {
+ log.warn("lookup failed for tablet " + entry.getKey(), e);
+ throw new RuntimeException(e);
+ }
+
+ bytesAdded += lookupResult.bytesAdded;
+
+ if (lookupResult.unfinishedRanges.size() > 0) {
+ if (lookupResult.closed) {
+ failures.put(entry.getKey(), lookupResult.unfinishedRanges);
+ } else {
+ session.queries.put(entry.getKey(), lookupResult.unfinishedRanges);
+ partScan = entry.getKey();
+ partNextKey = lookupResult.unfinishedRanges.get(0).getStartKey();
+ partNextKeyInclusive = lookupResult.unfinishedRanges.get(0).isStartKeyInclusive();
+ }
+ } else {
+ fullScans.add(entry.getKey());
+ }
+ }
+
+ long finishTime = System.currentTimeMillis();
+ session.totalLookupTime += (finishTime - startTime);
+ session.numEntries += results.size();
+
+ // convert everything to thrift before adding result
+ List<TKeyValue> retResults = new ArrayList<TKeyValue>();
+ for (KVEntry entry : results)
+ retResults.add(new TKeyValue(entry.key.toThrift(), ByteBuffer.wrap(entry.value)));
+ Map<TKeyExtent,List<TRange>> retFailures = Translator.translate(failures, Translators.KET, new Translator.ListTranslator<Range,TRange>(Translators.RT));
+ List<TKeyExtent> retFullScans = Translator.translate(fullScans, Translators.KET);
+ TKeyExtent retPartScan = null;
+ TKey retPartNextKey = null;
+ if (partScan != null) {
+ retPartScan = partScan.toThrift();
+ retPartNextKey = partNextKey.toThrift();
+ }
+ // add results to queue
+ addResult(new MultiScanResult(retResults, retFailures, retFullScans, retPartScan, retPartNextKey, partNextKeyInclusive, session.queries.size() != 0));
+ } catch (IterationInterruptedException iie) {
+ if (!isCancelled()) {
+ log.warn("Iteration interrupted, when scan not cancelled", iie);
+ addResult(iie);
+ }
+ } catch (Throwable e) {
+ log.warn("exception while doing multi-scan ", e);
+ addResult(e);
+ } finally {
+ Thread.currentThread().setName(oldThreadName);
+ runState.set(ScanRunState.FINISHED);
+ }
+ }
+ }
+
+ @Override
+ public InitialScan startScan(TInfo tinfo, TCredentials credentials, TKeyExtent textent, TRange range, List<TColumn> columns, int batchSize,
+ List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites, boolean isolated)
+ throws NotServingTabletException, ThriftSecurityException, org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
+
+ Authorizations userauths = null;
+ if (!security.canScan(credentials, new String(textent.getTable(), Constants.UTF8)))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ userauths = security.getUserAuthorizations(credentials);
+ for (ByteBuffer auth : authorizations)
+ if (!userauths.contains(ByteBufferUtil.toBytes(auth)))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
+
+ KeyExtent extent = new KeyExtent(textent);
+
+ // wait for any writes that are in flight.. this done to ensure
+ // consistency across client restarts... assume a client writes
+ // to accumulo and dies while waiting for a confirmation from
+ // accumulo... the client process restarts and tries to read
+ // data from accumulo making the assumption that it will get
+ // any writes previously made... however if the server side thread
+ // processing the write from the dead client is still in progress,
+ // the restarted client may not see the write unless we wait here.
+ // this behavior is very important when the client is reading the
+ // !METADATA table
+ if (waitForWrites)
+ writeTracker.waitForWrites(TabletType.type(extent));
+
+ Tablet tablet = onlineTablets.get(extent);
+ if (tablet == null)
+ throw new NotServingTabletException(textent);
+
+ ScanSession scanSession = new ScanSession();
+ scanSession.user = credentials.getPrincipal();
+ scanSession.extent = new KeyExtent(extent);
+ scanSession.columnSet = new HashSet<Column>();
+ scanSession.ssiList = ssiList;
+ scanSession.ssio = ssio;
+ scanSession.auths = new Authorizations(authorizations);
+ scanSession.interruptFlag = new AtomicBoolean();
+
+ for (TColumn tcolumn : columns) {
+ scanSession.columnSet.add(new Column(tcolumn));
+ }
+
+ scanSession.scanner = tablet.createScanner(new Range(range), batchSize, scanSession.columnSet, scanSession.auths, ssiList, ssio, isolated,
+ scanSession.interruptFlag);
+
+ long sid = sessionManager.createSession(scanSession, true);
+
+ ScanResult scanResult;
+ try {
+ scanResult = continueScan(tinfo, sid, scanSession);
+ } catch (NoSuchScanIDException e) {
+ log.error("The impossible happened", e);
+ throw new RuntimeException();
+ } finally {
+ sessionManager.unreserveSession(sid);
+ }
+
+ return new InitialScan(sid, scanResult);
+ }
+
+ @Override
+ public ScanResult continueScan(TInfo tinfo, long scanID) throws NoSuchScanIDException, NotServingTabletException,
+ org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
+ ScanSession scanSession = (ScanSession) sessionManager.reserveSession(scanID);
+ if (scanSession == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ try {
+ return continueScan(tinfo, scanID, scanSession);
+ } finally {
+ sessionManager.unreserveSession(scanSession);
+ }
+ }
+
+ private ScanResult continueScan(TInfo tinfo, long scanID, ScanSession scanSession) throws NoSuchScanIDException, NotServingTabletException,
+ org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
+
+ if (scanSession.nextBatchTask == null) {
+ scanSession.nextBatchTask = new NextBatchTask(scanID, scanSession.interruptFlag);
+ resourceManager.executeReadAhead(scanSession.extent, scanSession.nextBatchTask);
+ }
+
+ ScanBatch bresult;
+ try {
+ bresult = scanSession.nextBatchTask.get(MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS, TimeUnit.MILLISECONDS);
+ scanSession.nextBatchTask = null;
+ } catch (ExecutionException e) {
+ sessionManager.removeSession(scanID);
+ if (e.getCause() instanceof NotServingTabletException)
+ throw (NotServingTabletException) e.getCause();
+ else if (e.getCause() instanceof TooManyFilesException)
+ throw new org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException(scanSession.extent.toThrift());
+ else
+ throw new RuntimeException(e);
+ } catch (CancellationException ce) {
+ sessionManager.removeSession(scanID);
+ Tablet tablet = onlineTablets.get(scanSession.extent);
+ if (tablet == null || tablet.isClosed())
+ throw new NotServingTabletException(scanSession.extent.toThrift());
+ else
+ throw new NoSuchScanIDException();
+ } catch (TimeoutException e) {
+ List<TKeyValue> param = Collections.emptyList();
+ long timeout = acuConf.getTimeInMillis(Property.TSERV_CLIENT_TIMEOUT);
+ sessionManager.removeIfNotAccessed(scanID, timeout);
+ return new ScanResult(param, true);
+ } catch (Throwable t) {
+ sessionManager.removeSession(scanID);
+ log.warn("Failed to get next batch", t);
+ throw new RuntimeException(t);
+ }
+
+ ScanResult scanResult = new ScanResult(Key.compress(bresult.results), bresult.more);
+
+ scanSession.entriesReturned += scanResult.results.size();
+
+ scanSession.batchCount++;
+
+ if (scanResult.more && scanSession.batchCount > 3) {
+ // start reading next batch while current batch is transmitted
+ // to client
+ scanSession.nextBatchTask = new NextBatchTask(scanID, scanSession.interruptFlag);
+ resourceManager.executeReadAhead(scanSession.extent, scanSession.nextBatchTask);
+ }
+
+ if (!scanResult.more)
+ closeScan(tinfo, scanID);
+
+ return scanResult;
+ }
+
+ @Override
+ public void closeScan(TInfo tinfo, long scanID) {
+ ScanSession ss = (ScanSession) sessionManager.removeSession(scanID);
+ if (ss != null) {
+ long t2 = System.currentTimeMillis();
+
+ log.debug(String.format("ScanSess tid %s %s %,d entries in %.2f secs, nbTimes = [%s] ", TServerUtils.clientAddress.get(), ss.extent.getTableId()
+ .toString(), ss.entriesReturned, (t2 - ss.startTime) / 1000.0, ss.nbTimes.toString()));
+ if (scanMetrics.isEnabled()) {
+ scanMetrics.add(TabletServerScanMetrics.scan, t2 - ss.startTime);
+ scanMetrics.add(TabletServerScanMetrics.resultSize, ss.entriesReturned);
+ }
+ }
+ }
+
+ @Override
+ public InitialMultiScan startMultiScan(TInfo tinfo, TCredentials credentials, Map<TKeyExtent,List<TRange>> tbatch, List<TColumn> tcolumns,
+ List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites) throws ThriftSecurityException {
+ // find all of the tables that need to be scanned
+ HashSet<String> tables = new HashSet<String>();
+ for (TKeyExtent keyExtent : tbatch.keySet()) {
+ tables.add(new String(keyExtent.getTable(), Constants.UTF8));
+ }
+
+ // check if user has permission to the tables
+ Authorizations userauths = null;
+ for (String table : tables)
+ if (!security.canScan(credentials, table))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ userauths = security.getUserAuthorizations(credentials);
+ for (ByteBuffer auth : authorizations)
+ if (!userauths.contains(ByteBufferUtil.toBytes(auth)))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
+
+ KeyExtent threadPoolExtent = null;
+
+ Map<KeyExtent,List<Range>> batch = Translator.translate(tbatch, Translators.TKET, new Translator.ListTranslator<TRange,Range>(Translators.TRT));
+
+ for (KeyExtent keyExtent : batch.keySet()) {
+ if (threadPoolExtent == null) {
+ threadPoolExtent = keyExtent;
+ } else if (keyExtent.isRootTablet()) {
+ throw new IllegalArgumentException("Cannot batch query root tablet with other tablets " + threadPoolExtent + " " + keyExtent);
+ } else if (keyExtent.isMeta() && !threadPoolExtent.isMeta()) {
+ throw new IllegalArgumentException("Cannot batch query !METADATA and non !METADATA tablets " + threadPoolExtent + " " + keyExtent);
+ }
+
+ }
+
+ if (waitForWrites)
+ writeTracker.waitForWrites(TabletType.type(batch.keySet()));
+
+ MultiScanSession mss = new MultiScanSession();
+ mss.user = credentials.getPrincipal();
+ mss.queries = batch;
+ mss.columnSet = new HashSet<Column>(tcolumns.size());
+ mss.ssiList = ssiList;
+ mss.ssio = ssio;
+ mss.auths = new Authorizations(authorizations);
+
+ mss.numTablets = batch.size();
+ for (List<Range> ranges : batch.values()) {
+ mss.numRanges += ranges.size();
+ }
+
+ for (TColumn tcolumn : tcolumns)
+ mss.columnSet.add(new Column(tcolumn));
+
+ mss.threadPoolExtent = threadPoolExtent;
+
+ long sid = sessionManager.createSession(mss, true);
+
+ MultiScanResult result;
+ try {
+ result = continueMultiScan(tinfo, sid, mss);
+ } catch (NoSuchScanIDException e) {
+ log.error("the impossible happened", e);
+ throw new RuntimeException("the impossible happened", e);
+ } finally {
+ sessionManager.unreserveSession(sid);
+ }
+
+ return new InitialMultiScan(sid, result);
+ }
+
+ @Override
+ public MultiScanResult continueMultiScan(TInfo tinfo, long scanID) throws NoSuchScanIDException {
+
+ MultiScanSession session = (MultiScanSession) sessionManager.reserveSession(scanID);
+
+ if (session == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ try {
+ return continueMultiScan(tinfo, scanID, session);
+ } finally {
+ sessionManager.unreserveSession(session);
+ }
+ }
+
+ private MultiScanResult continueMultiScan(TInfo tinfo, long scanID, MultiScanSession session) throws NoSuchScanIDException {
+
+ if (session.lookupTask == null) {
+ session.lookupTask = new LookupTask(scanID);
+ resourceManager.executeReadAhead(session.threadPoolExtent, session.lookupTask);
+ }
+
+ try {
+ MultiScanResult scanResult = session.lookupTask.get(MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS, TimeUnit.MILLISECONDS);
+ session.lookupTask = null;
+ return scanResult;
+ } catch (TimeoutException e1) {
+ long timeout = acuConf.getTimeInMillis(Property.TSERV_CLIENT_TIMEOUT);
+ sessionManager.removeIfNotAccessed(scanID, timeout);
+ List<TKeyValue> results = Collections.emptyList();
+ Map<TKeyExtent,List<TRange>> failures = Collections.emptyMap();
+ List<TKeyExtent> fullScans = Collections.emptyList();
+ return new MultiScanResult(results, failures, fullScans, null, null, false, true);
+ } catch (Throwable t) {
+ sessionManager.removeSession(scanID);
+ log.warn("Failed to get multiscan result", t);
+ throw new RuntimeException(t);
+ }
+ }
+
+ @Override
+ public void closeMultiScan(TInfo tinfo, long scanID) throws NoSuchScanIDException {
+ MultiScanSession session = (MultiScanSession) sessionManager.removeSession(scanID);
+ if (session == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ long t2 = System.currentTimeMillis();
+ log.debug(String.format("MultiScanSess %s %,d entries in %.2f secs (lookup_time:%.2f secs tablets:%,d ranges:%,d) ", TServerUtils.clientAddress.get(),
+ session.numEntries, (t2 - session.startTime) / 1000.0, session.totalLookupTime / 1000.0, session.numTablets, session.numRanges));
+ }
+
+ @Override
+ public long startUpdate(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException {
+ // Make sure user is real
+
+ security.authenticateUser(credentials, credentials);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
+
+ UpdateSession us = new UpdateSession();
+ us.violations = new Violations();
+ us.credentials = credentials;
+ us.cenv = new TservConstraintEnv(security, us.credentials);
+
+ long sid = sessionManager.createSession(us, false);
+
+ return sid;
+ }
+
+ private void setUpdateTablet(UpdateSession us, KeyExtent keyExtent) {
+ long t1 = System.currentTimeMillis();
+ if (us.currentTablet != null && us.currentTablet.getExtent().equals(keyExtent))
+ return;
+ if (us.currentTablet == null && (us.failures.containsKey(keyExtent) || us.authFailures.containsKey(keyExtent))) {
+ // if there were previous failures, then do not accept additional writes
+ return;
+ }
+
+ try {
+ // if user has no permission to write to this table, add it to
+ // the failures list
+ boolean sameTable = us.currentTablet != null && (us.currentTablet.getExtent().getTableId().equals(keyExtent.getTableId()));
+ if (sameTable || security.canWrite(us.credentials, keyExtent.getTableId().toString())) {
+ long t2 = System.currentTimeMillis();
+ us.authTimes.addStat(t2 - t1);
+ us.currentTablet = onlineTablets.get(keyExtent);
+ if (us.currentTablet != null) {
+ us.queuedMutations.put(us.currentTablet, new ArrayList<Mutation>());
+ } else {
+ // not serving tablet, so report all mutations as
+ // failures
+ us.failures.put(keyExtent, 0l);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.unknownTabletErrors, 0);
+ }
+ } else {
+ log.warn("Denying access to table " + keyExtent.getTableId() + " for user " + us.credentials.getPrincipal());
+ long t2 = System.currentTimeMillis();
+ us.authTimes.addStat(t2 - t1);
+ us.currentTablet = null;
+ us.authFailures.put(keyExtent, SecurityErrorCode.PERMISSION_DENIED);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
+ return;
+ }
+ } catch (ThriftSecurityException e) {
+ log.error("Denying permission to check user " + us.credentials.getPrincipal() + " with user " + e.getUser(), e);
+ long t2 = System.currentTimeMillis();
+ us.authTimes.addStat(t2 - t1);
+ us.currentTablet = null;
+ us.authFailures.put(keyExtent, e.getCode());
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
+ return;
+ }
+ }
+
+ @Override
+ public void applyUpdates(TInfo tinfo, long updateID, TKeyExtent tkeyExtent, List<TMutation> tmutations) {
+ UpdateSession us = (UpdateSession) sessionManager.reserveSession(updateID);
+ if (us == null) {
+ throw new RuntimeException("No Such SessionID");
+ }
+
+ try {
+ KeyExtent keyExtent = new KeyExtent(tkeyExtent);
+ setUpdateTablet(us, keyExtent);
+
+ if (us.currentTablet != null) {
+ List<Mutation> mutations = us.queuedMutations.get(us.currentTablet);
+ for (TMutation tmutation : tmutations) {
+ Mutation mutation = new ServerMutation(tmutation);
+ mutations.add(mutation);
+ us.queuedMutationSize += mutation.numBytes();
+ }
+ if (us.queuedMutationSize > getSystemConfiguration().getMemoryInBytes(Property.TSERV_MUTATION_QUEUE_MAX))
+ flush(us);
+ }
+ } finally {
+ sessionManager.unreserveSession(us);
+ }
+ }
+
+ private void flush(UpdateSession us) {
+
+ int mutationCount = 0;
+ Map<CommitSession,List<Mutation>> sendables = new HashMap<CommitSession,List<Mutation>>();
+ Throwable error = null;
+
+ long pt1 = System.currentTimeMillis();
+
+ boolean containsMetadataTablet = false;
+ for (Tablet tablet : us.queuedMutations.keySet())
+ if (tablet.getExtent().isMeta())
+ containsMetadataTablet = true;
+
+ if (!containsMetadataTablet && us.queuedMutations.size() > 0)
+ TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
+
+ Span prep = Trace.start("prep");
+ for (Entry<Tablet,? extends List<Mutation>> entry : us.queuedMutations.entrySet()) {
+
+ Tablet tablet = entry.getKey();
+ List<Mutation> mutations = entry.getValue();
+ if (mutations.size() > 0) {
+ try {
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.mutationArraySize, mutations.size());
+
+ CommitSession commitSession = tablet.prepareMutationsForCommit(us.cenv, mutations);
+ if (commitSession == null) {
+ if (us.currentTablet == tablet) {
+ us.currentTablet = null;
+ }
+ us.failures.put(tablet.getExtent(), us.successfulCommits.get(tablet));
+ } else {
+ sendables.put(commitSession, mutations);
+ mutationCount += mutations.size();
+ }
+
+ } catch (TConstraintViolationException e) {
+ us.violations.add(e.getViolations());
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.constraintViolations, 0);
+
+ if (e.getNonViolators().size() > 0) {
+ // only log and commit mutations if there were some
+ // that did not
+ // violate constraints... this is what
+ // prepareMutationsForCommit()
+ // expects
+ sendables.put(e.getCommitSession(), e.getNonViolators());
+ }
+
+ mutationCount += mutations.size();
+
+ } catch (HoldTimeoutException t) {
+ error = t;
+ log.debug("Giving up on mutations due to a long memory hold time");
+ break;
+ } catch (Throwable t) {
+ error = t;
+ log.error("Unexpected error preparing for commit", error);
+ break;
+ }
+ }
+ }
+ prep.stop();
+
+ Span wal = Trace.start("wal");
+ long pt2 = System.currentTimeMillis();
+ long avgPrepareTime = (long) ((pt2 - pt1) / (double) us.queuedMutations.size());
+ us.prepareTimes.addStat(pt2 - pt1);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.commitPrep, (avgPrepareTime));
+
+ if (error != null) {
+ for (Entry<CommitSession,List<Mutation>> e : sendables.entrySet()) {
+ e.getKey().abortCommit(e.getValue());
+ }
+ throw new RuntimeException(error);
+ }
+ try {
+ while (true) {
+ try {
+ long t1 = System.currentTimeMillis();
+
+ logger.logManyTablets(sendables);
+
+ long t2 = System.currentTimeMillis();
+ us.walogTimes.addStat(t2 - t1);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.waLogWriteTime, (t2 - t1));
+
+ break;
+ } catch (IOException ex) {
+ log.warn("logging mutations failed, retrying");
+ } catch (Throwable t) {
+ log.error("Unknown exception logging mutations, counts for mutations in flight not decremented!", t);
+ throw new RuntimeException(t);
+ }
+ }
+
+ wal.stop();
+
+ Span commit = Trace.start("commit");
+ long t1 = System.currentTimeMillis();
+ for (Entry<CommitSession,? extends List<Mutation>> entry : sendables.entrySet()) {
+ CommitSession commitSession = entry.getKey();
+ List<Mutation> mutations = entry.getValue();
+
+ commitSession.commit(mutations);
+
+ Tablet tablet = commitSession.getTablet();
+
+ if (tablet == us.currentTablet) {
+ // because constraint violations may filter out some
+ // mutations, for proper
+ // accounting with the client code, need to increment
+ // the count based
+ // on the original number of mutations from the client
+ // NOT the filtered number
+ us.successfulCommits.increment(tablet, us.queuedMutations.get(tablet).size());
+ }
+ }
+ long t2 = System.currentTimeMillis();
+
+ long avgCommitTime = (long) ((t2 - t1) / (double) sendables.size());
+
+ us.flushTime += (t2 - pt1);
+ us.commitTimes.addStat(t2 - t1);
+
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.commitTime, avgCommitTime);
+ commit.stop();
+ } finally {
+ us.queuedMutations.clear();
+ if (us.currentTablet != null) {
+ us.queuedMutations.put(us.currentTablet, new ArrayList<Mutation>());
+ }
+ us.queuedMutationSize = 0;
+ }
+ us.totalUpdates += mutationCount;
+ }
+
+ @Override
+ public UpdateErrors closeUpdate(TInfo tinfo, long updateID) throws NoSuchScanIDException {
+ UpdateSession us = (UpdateSession) sessionManager.removeSession(updateID);
+ if (us == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ // clients may or may not see data from an update session while
+ // it is in progress, however when the update session is closed
+ // want to ensure that reads wait for the write to finish
+ long opid = writeTracker.startWrite(us.queuedMutations.keySet());
+
+ try {
+ flush(us);
+ } finally {
+ writeTracker.finishWrite(opid);
+ }
+
+ log.debug(String.format("UpSess %s %,d in %.3fs, at=[%s] ft=%.3fs(pt=%.3fs lt=%.3fs ct=%.3fs)", TServerUtils.clientAddress.get(), us.totalUpdates,
+ (System.currentTimeMillis() - us.startTime) / 1000.0, us.authTimes.toString(), us.flushTime / 1000.0, us.prepareTimes.getSum() / 1000.0,
+ us.walogTimes.getSum() / 1000.0, us.commitTimes.getSum() / 1000.0));
+ if (us.failures.size() > 0) {
+ Entry<KeyExtent,Long> first = us.failures.entrySet().iterator().next();
+ log.debug(String.format("Failures: %d, first extent %s successful commits: %d", us.failures.size(), first.getKey().toString(), first.getValue()));
+ }
+ List<ConstraintViolationSummary> violations = us.violations.asList();
+ if (violations.size() > 0) {
+ ConstraintViolationSummary first = us.violations.asList().iterator().next();
+ log.debug(String.format("Violations: %d, first %s occurs %d", violations.size(), first.violationDescription, first.numberOfViolatingMutations));
+ }
+ if (us.authFailures.size() > 0) {
+ KeyExtent first = us.authFailures.keySet().iterator().next();
+ log.debug(String.format("Authentication Failures: %d, first %s", us.authFailures.size(), first.toString()));
+ }
+
+ return new UpdateErrors(Translator.translate(us.failures, Translators.KET), Translator.translate(violations, Translators.CVST), Translator.translate(
+ us.authFailures, Translators.KET));
+ }
+
+ @Override
+ public void update(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, TMutation tmutation) throws NotServingTabletException,
+ ConstraintViolationException, ThriftSecurityException {
+
+ if (!security.canWrite(credentials, new String(tkeyExtent.getTable(), Constants.UTF8)))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ KeyExtent keyExtent = new KeyExtent(tkeyExtent);
+ Tablet tablet = onlineTablets.get(new KeyExtent(keyExtent));
+ if (tablet == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+
+ if (!keyExtent.isMeta())
+ TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
+
+ long opid = writeTracker.startWrite(TabletType.type(keyExtent));
+
+ try {
+ Mutation mutation = new ServerMutation(tmutation);
+ List<Mutation> mutations = Collections.singletonList(mutation);
+
+ Span prep = Trace.start("prep");
+ CommitSession cs = tablet.prepareMutationsForCommit(new TservConstraintEnv(security, credentials), mutations);
+ prep.stop();
+ if (cs == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+
+ while (true) {
+ try {
+ Span wal = Trace.start("wal");
+ logger.log(cs, cs.getWALogSeq(), mutation);
+ wal.stop();
+ break;
+ } catch (IOException ex) {
+ log.warn(ex, ex);
+ }
+ }
+
+ Span commit = Trace.start("commit");
+ cs.commit(mutations);
+ commit.stop();
+ } catch (TConstraintViolationException e) {
+ throw new ConstraintViolationException(Translator.translate(e.getViolations().asList(), Translators.CVST));
+ } finally {
+ writeTracker.finishWrite(opid);
+ }
+ }
+
+ @Override
+ public void splitTablet(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, ByteBuffer splitPoint)
+ throws NotServingTabletException, ThriftSecurityException {
+
+ String tableId = new String(ByteBufferUtil.toBytes(tkeyExtent.table), Constants.UTF8);
+ if (!security.canSplitTablet(credentials, tableId))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ KeyExtent keyExtent = new KeyExtent(tkeyExtent);
+
+ Tablet tablet = onlineTablets.get(keyExtent);
+ if (tablet == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+
+ if (keyExtent.getEndRow() == null || !keyExtent.getEndRow().equals(ByteBufferUtil.toText(splitPoint))) {
+ try {
+ if (TabletServer.this.splitTablet(tablet, ByteBufferUtil.toBytes(splitPoint)) == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+ } catch (IOException e) {
+ log.warn("Failed to split " + keyExtent, e);
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Override
+ public TabletServerStatus getTabletServerStatus(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ return getStats(sessionManager.getActiveScansPerTable());
+ }
+
+ @Override
+ public List<TabletStats> getTabletStats(TInfo tinfo, TCredentials credentials, String tableId) throws ThriftSecurityException, TException {
+ TreeMap<KeyExtent,Tablet> onlineTabletsCopy;
+ synchronized (onlineTablets) {
+ onlineTabletsCopy = new TreeMap<KeyExtent,Tablet>(onlineTablets);
+ }
+ List<TabletStats> result = new ArrayList<TabletStats>();
+ Text text = new Text(tableId);
+ KeyExtent start = new KeyExtent(text, new Text(), null);
+ for (Entry<KeyExtent,Tablet> entry : onlineTabletsCopy.tailMap(start).entrySet()) {
+ KeyExtent ke = entry.getKey();
+ if (ke.getTableId().compareTo(text) == 0) {
+ Tablet tablet = entry.getValue();
+ TabletStats stats = tablet.timer.getTabletStats();
+ stats.extent = ke.toThrift();
+ stats.ingestRate = tablet.ingestRate();
+ stats.queryRate = tablet.queryRate();
+ stats.splitCreationTime = tablet.getSplitCreationTime();
+ stats.numEntries = tablet.getNumEntries();
+ result.add(stats);
+ }
+ }
+ return result;
+ }
+
+ private ZooCache masterLockCache = new ZooCache();
+
+ private void checkPermission(TCredentials credentials, String lock, boolean requiresSystemPermission, final String request)
+ throws ThriftSecurityException {
+ if (requiresSystemPermission) {
+ boolean fatal = false;
+ try {
+ log.debug("Got " + request + " message from user: " + credentials.getPrincipal());
+ if (!security.canPerformSystemActions(credentials)) {
+ log.warn("Got " + request + " message from user: " + credentials.getPrincipal());
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ }
+ } catch (ThriftSecurityException e) {
+ log.warn("Got " + request + " message from unauthenticatable user: " + e.getUser());
+ if (e.getUser().equals(SecurityConstants.SYSTEM_PRINCIPAL)) {
+ log.fatal("Got message from a service with a mismatched configuration. Please ensure a compatible configuration.", e);
+ fatal = true;
+ }
+ throw e;
+ } finally {
+ if (fatal) {
+ Halt.halt(1, new Runnable() {
+ @Override
+ public void run() {
+ logGCInfo(getSystemConfiguration());
+ }
+ });
+ }
+ }
+ }
+
+ if (tabletServerLock == null || !tabletServerLock.wasLockAcquired()) {
+ log.warn("Got " + request + " message from master before lock acquired, ignoring...");
+ throw new RuntimeException("Lock not acquired");
+ }
+
+ if (tabletServerLock != null && tabletServerLock.wasLockAcquired() && !tabletServerLock.isLocked()) {
+ Halt.halt(1, new Runnable() {
+ @Override
+ public void run() {
+ log.info("Tablet server no longer holds lock during checkPermission() : " + request + ", exiting");
+ logGCInfo(getSystemConfiguration());
+ }
+ });
+ }
+
+ if (lock != null) {
+ ZooUtil.LockID lid = new ZooUtil.LockID(ZooUtil.getRoot(instance) + Constants.ZMASTER_LOCK, lock);
+
+ try {
+ if (!ZooLock.isLockHeld(masterLockCache, lid)) {
+ // maybe the cache is out of date and a new master holds the
+ // lock?
+ masterLockCache.clear();
+ if (!ZooLock.isLockHeld(masterLockCache, lid)) {
+ log.warn("Got " + request + " message from a master that does not hold the current lock " + lock);
+ throw new RuntimeException("bad master lock");
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("bad master lock", e);
+ }
+ }
+ }
+
+ @Override
+ public void loadTablet(TInfo tinfo, TCredentials credentials, String lock, final TKeyExtent textent) {
+
+ try {
+ checkPermission(credentials, lock, true, "loadTablet");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+
+ final KeyExtent extent = new KeyExtent(textent);
+
+ synchronized (unopenedTablets) {
+ synchronized (openingTablets) {
+ synchronized (onlineTablets) {
+
+ // checking if this exact tablet is in any of the sets
+ // below is not a strong enough check
+ // when splits and fix splits occurring
+
+ Set<KeyExtent> unopenedOverlapping = KeyExtent.findOverlapping(extent, unopenedTablets);
+ Set<KeyExtent> openingOverlapping = KeyExtent.findOverlapping(extent, openingTablets);
+ Set<KeyExtent> onlineOverlapping = KeyExtent.findOverlapping(extent, onlineTablets);
+ Set<KeyExtent> all = new HashSet<KeyExtent>();
+ all.addAll(unopenedOverlapping);
+ all.addAll(openingOverlapping);
+ all.addAll(onlineOverlapping);
+
+ if (!all.isEmpty()) {
+ if (all.size() != 1 || !all.contains(extent)) {
+ log.error("Tablet " + extent + " overlaps previously assigned " + unopenedOverlapping + " " + openingOverlapping + " " + onlineOverlapping);
+ }
+ return;
+ }
+
+ unopenedTablets.add(extent);
+ }
+ }
+ }
+
+ // add the assignment job to the appropriate queue
+ log.info("Loading tablet " + extent);
+
+ final Runnable ah = new LoggingRunnable(log, new AssignmentHandler(extent));
+ // Root tablet assignment must take place immediately
+ if (extent.isRootTablet()) {
+ new Daemon("Root Tablet Assignment") {
+ @Override
+ public void run() {
+ ah.run();
+ if (onlineTablets.containsKey(extent)) {
+ log.info("Root tablet loaded: " + extent);
+ } else {
+ log.info("Root tablet failed to load");
+ }
+
+ }
+ }.start();
+ } else {
+ if (extent.isMeta()) {
+ resourceManager.addMetaDataAssignment(ah);
+ } else {
+ resourceManager.addAssignment(ah);
+ }
+ }
+ }
+
+ @Override
+ public void unloadTablet(TInfo tinfo, TCredentials credentials, String lock, TKeyExtent textent, boolean save) {
+ try {
+ checkPermission(credentials, lock, true, "unloadTablet");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+
+ KeyExtent extent = new KeyExtent(textent);
+
+ resourceManager.addMigration(extent, new LoggingRunnable(log, new UnloadTabletHandler(extent, save)));
+ }
+
+ @Override
+ public void flush(TInfo tinfo, TCredentials credentials, String lock, String tableId, ByteBuffer startRow, ByteBuffer endRow) {
+ try {
+ checkPermission(credentials, lock, true, "flush");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+
+ ArrayList<Tablet> tabletsToFlush = new ArrayList<Tablet>();
+
+ KeyExtent ke = new KeyExtent(new Text(tableId), ByteBufferUtil.toText(endRow), ByteBufferUtil.toText(startRow));
+
+ synchronized (onlineTablets) {
+ for (Tablet tablet : onlineTablets.values())
+ if (ke.overlaps(tablet.getExtent()))
+ tabletsToFlush.add(tablet);
+ }
+
+ Long flushID = null;
+
+ for (Tablet tablet : tabletsToFlush) {
+ if (flushID == null) {
+ // read the flush id once from zookeeper instead of reading
+ // it for each tablet
+ try {
+ flushID = tablet.getFlushID();
+ } catch (NoNodeException e) {
+ // table was probably deleted
+ log.info("Asked to flush table that has no flush id " + ke + " " + e.getMessage());
+ return;
+ }
+ }
+ tablet.flush(flushID);
+ }
+ }
+
+ @Override
+ public void flushTablet(TInfo tinfo, TCredentials credentials, String lock, TKeyExtent textent) throws TException {
+ try {
+ checkPermission(credentials, lock, true, "flushTablet");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+
+ Tablet tablet = onlineTablets.get(new KeyExtent(textent));
+ if (tablet != null) {
+ log.info("Flushing " + tablet.getExtent());
+ try {
+ tablet.flush(tablet.getFlushID());
+ } catch (NoNodeException nne) {
+ log.info("Asked to flush tablet that has no flush id " + new KeyExtent(textent) + " " + nne.getMessage());
+ }
+ }
+ }
+
+ @Override
+ public void halt(TInfo tinfo, TCredentials credentials, String lock) throws ThriftSecurityException {
+
+ checkPermission(credentials, lock, true, "halt");
+
+ Halt.halt(0, new Runnable() {
+ @Override
+ public void run() {
+ log.info("Master requested tablet server halt");
+ logGCInfo(getSystemConfiguration());
+ serverStopRequested = true;
+ try {
+ tabletServerLock.unlock();
+ } catch (Exception e) {
+ log.error(e, e);
+ }
+ }
+ });
+ }
+
+ @Override
+ public void fastHalt(TInfo info, TCredentials credentials, String lock) {
+ try {
+ halt(info, credentials, lock);
+ } catch (Exception e) {
+ log.warn("Error halting", e);
+ }
+ }
+
+ @Override
+ public TabletStats getHistoricalStats(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ return statsKeeper.getTabletStats();
+ }
+
+ @Override
+ public List<ActiveScan> getActiveScans(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ try {
+ checkPermission(credentials, null, true, "getScans");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw e;
+ }
+
+ return sessionManager.getActiveScans();
+ }
+
+ @Override
+ public void chop(TInfo tinfo, TCredentials credentials, String lock, TKeyExtent textent) throws TException {
+ try {
+ checkPermission(credentials, lock, true, "chop");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+
+ KeyExtent ke = new KeyExtent(textent);
+
+ Tablet tablet = onlineTablets.get(ke);
+ if (tablet != null) {
+ tablet.chopFiles();
+ }
+ }
+
+ @Override
+ public void compact(TInfo tinfo, TCredentials credentials, String lock, String tableId, ByteBuffer startRow, ByteBuffer endRow)
+ throws TException {
+ try {
+ checkPermission(credentials, lock, true, "compact");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+
+ KeyExtent ke = new KeyExtent(new Text(tableId), ByteBufferUtil.toText(endRow), ByteBufferUtil.toText(startRow));
+
+ ArrayList<Tablet> tabletsToCompact = new ArrayList<Tablet>();
+ synchronized (onlineTablets) {
+ for (Tablet tablet : onlineTablets.values())
+ if (ke.overlaps(tablet.getExtent()))
+ tabletsToCompact.add(tablet);
+ }
+
+ Long compactionId = null;
+
+ for (Tablet tablet : tabletsToCompact) {
+ // all for the same table id, so only need to read
+ // compaction id once
+ if (compactionId == null)
+ try {
+ compactionId = tablet.getCompactionID().getFirst();
+ } catch (NoNodeException e) {
+ log.info("Asked to compact table with no compaction id " + ke + " " + e.getMessage());
+ return;
+ }
+ tablet.compactAll(compactionId);
+ }
+
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface#removeLogs(org.apache.accumulo.trace.thrift.TInfo,
- * org.apache.accumulo.core.security.thrift.Credentials, java.util.List)
- */
+ @Override
+ public void removeLogs(TInfo tinfo, TCredentials credentials, List<String> filenames) throws TException {
+ String myname = getClientAddressString();
+ myname = myname.replace(':', '+');
+ Path logDir = new Path(Constants.getWalDirectory(acuConf), myname);
+ Set<String> loggers = new HashSet<String>();
+ logger.getLoggers(loggers);
+ nextFile: for (String filename : filenames) {
+ for (String logger : loggers) {
+ if (logger.contains(filename))
+ continue nextFile;
+ }
+ List<Tablet> onlineTabletsCopy = new ArrayList<Tablet>();
+ synchronized (onlineTablets) {
+ onlineTabletsCopy.addAll(onlineTablets.values());
+ }
+ for (Tablet tablet : onlineTabletsCopy) {
+ for (String current : tablet.getCurrentLogs()) {
+ if (current.contains(filename)) {
+ log.info("Attempted to delete " + filename + " from tablet " + tablet.getExtent());
+ continue nextFile;
+ }
+ }
+ }
+ try {
+ String source = logDir + "/" + filename;
+ if (acuConf.getBoolean(Property.TSERV_ARCHIVE_WALOGS)) {
+ String walogArchive = Constants.getBaseDir(acuConf) + "/walogArchive";
+ fs.mkdirs(new Path(walogArchive));
+ String dest = walogArchive + "/" + filename;
+ log.info("Archiving walog " + source + " to " + dest);
+ if (!fs.rename(new Path(source), new Path(dest)))
+ log.error("rename is unsuccessful");
+ } else {
+ log.info("Deleting walog " + filename);
+ Trash trash = new Trash(fs, fs.getConf());
+ Path sourcePath = new Path(source);
+ if (!(!acuConf.getBoolean(Property.GC_TRASH_IGNORE) && trash.moveToTrash(sourcePath)) && !fs.delete(sourcePath, true))
+ log.warn("Failed to delete walog " + source);
+ Path recoveryPath = new Path(Constants.getRecoveryDir(acuConf), filename);
+ try {
+ if (trash.moveToTrash(recoveryPath) || fs.delete(recoveryPath, true))
+ log.info("Deleted any recovery log " + filename);
+ } catch (FileNotFoundException ex) {
+ // ignore
+ }
+
+ }
+ } catch (IOException e) {
+ log.warn("Error attempting to delete write-ahead log " + filename + ": " + e);
+ }
+ }
+ }
+
+ @Override
+ public List<ActiveCompaction> getActiveCompactions(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ try {
+ checkPermission(credentials, null, true, "getActiveCompactions");
+ } catch (ThriftSecurityException e) {
+ log.error(e, e);
+ throw e;
+ }
+
+ List<CompactionInfo> compactions = Compactor.getRunningCompactions();
+ List<ActiveCompaction> ret = new ArrayList<ActiveCompaction>(compactions.size());
+
+ for (CompactionInfo compactionInfo : compactions) {
+ ret.add(compactionInfo.toThrift());
+ }
+
+ return ret;
+ }
+ }
+
+ private class SplitRunner implements Runnable {
+ private Tablet tablet;
+
+ public SplitRunner(Tablet tablet) {
+ this.tablet = tablet;
+ }
+
+ @Override
+ public void run() {
+ if (majorCompactorDisabled) {
+ // this will make split task that were queued when shutdown was
+ // initiated exit
+ return;
+ }
+
+ splitTablet(tablet);
+ }
+ }
+
+ boolean isMajorCompactionDisabled() {
+ return majorCompactorDisabled;
+ }
+
+ void executeSplit(Tablet tablet) {
+ resourceManager.executeSplit(tablet.getExtent(), new LoggingRunnable(log, new SplitRunner(tablet)));
+ }
+
+ private class MajorCompactor implements Runnable {
+
+ @Override
+ public void run() {
+ while (!majorCompactorDisabled) {
+ try {
+ UtilWaitThread.sleep(getSystemConfiguration().getTimeInMillis(Property.TSERV_MAJC_DELAY));
+
+ TreeMap<KeyExtent,Tablet> copyOnlineTablets = new TreeMap<KeyExtent,Tablet>();
+
+ synchronized (onlineTablets) {
+ copyOnlineTablets.putAll(onlineTablets); // avoid
+ // concurrent
+ // modification
+ }
+
+ int numMajorCompactions
<TRUNCATED>
[52/64] [abbrv] Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/CompactionStrategy.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/CompactionStrategy.java
index 52688cb,0000000..7bc1a80
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/CompactionStrategy.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/CompactionStrategy.java
@@@ -1,71 -1,0 +1,67 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver.compaction;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * The interface for customizing major compactions.
+ * <p>
+ * The tablet server has one thread to ask many tablets if they should compact. When the strategy returns true, then tablet is added to the queue of tablets
+ * waiting for a compaction thread. Once a thread is available, the {@link #gatherInformation(MajorCompactionRequest)} method is called outside the tablets'
+ * lock. This gives the strategy the ability to read information that maybe expensive to fetch. Once the gatherInformation returns, the tablet lock is grabbed
+ * and the compactionPlan computed. This should *not* do expensive operations, especially not I/O. Note that the number of files may change between calls to
+ * {@link #gatherInformation(MajorCompactionRequest)} and {@link #getCompactionPlan(MajorCompactionRequest)}.
+ * <p>
+ * <b>Note:</b> the strategy object used for the {@link #shouldCompact(MajorCompactionRequest)} call is going to be different from the one used in the
+ * compaction thread.
+ */
+public abstract class CompactionStrategy {
+
+ /**
+ * The settings for the compaction strategy pulled from zookeeper. The <tt>table.compacations.major.strategy.opts</tt> part of the setting will be removed.
- *
- * @param options
+ */
+ public void init(Map<String,String> options) {}
+
+ /**
+ * Determine if this tablet is eligible for a major compaction. It's ok if it later determines (through {@link #gatherInformation(MajorCompactionRequest)} and
+ * {@link #getCompactionPlan(MajorCompactionRequest)}) that it does not need to. Any state stored during shouldCompact will no longer exist when
+ * {@link #gatherInformation(MajorCompactionRequest)} and {@link #getCompactionPlan(MajorCompactionRequest)} are called.
+ *
+ */
+ public abstract boolean shouldCompact(MajorCompactionRequest request) throws IOException;
+
+ /**
+ * Called prior to obtaining the tablet lock, useful for examining metadata or indexes. State collected during this method will be available during the call
+ * the {@link #getCompactionPlan(MajorCompactionRequest)}.
+ *
+ * @param request
+ * basic details about the tablet
- * @throws IOException
+ */
+ public void gatherInformation(MajorCompactionRequest request) throws IOException {}
+
+ /**
+ * Get the plan for compacting a tablets files. Called while holding the tablet lock, so it should not be doing any blocking.
+ *
+ * @param request
+ * basic details about the tablet
+ * @return the plan for a major compaction, or null to cancel the compaction.
- * @throws IOException
+ */
+ abstract public CompactionPlan getCompactionPlan(MajorCompactionRequest request) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
index 25b8043,0000000..a1229e7
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
@@@ -1,170 -1,0 +1,169 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver.logger;
+
+import java.io.DataInputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.tserver.log.DfsLogger;
+import org.apache.accumulo.tserver.log.DfsLogger.DFSLoggerInputStreams;
+import org.apache.accumulo.tserver.log.MultiReader;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+
+public class LogReader {
+
+ static class Opts extends Help {
+ @Parameter(names = "-r", description = "print only mutations associated with the given row")
+ String row;
+ @Parameter(names = "-m", description = "limit the number of mutations printed per row")
+ int maxMutations = 5;
+ @Parameter(names = "-t", description = "print only mutations that fall within the given key extent")
+ String extent;
+ @Parameter(names = "-p", description = "search for a row that matches the given regex")
+ String regexp;
+ @Parameter(description = "<logfile> { <logfile> ... }")
+ List<String> files = new ArrayList<String>();
+ }
+
+ /**
+ * Dump a Log File (Map or Sequence) to stdout. Will read from HDFS or local file system.
+ *
+ * @param args
+ * - first argument is the file to print
- * @throws IOException
+ */
+ public static void main(String[] args) throws IOException {
+ Opts opts = new Opts();
+ opts.parseArgs(LogReader.class.getName(), args);
+ VolumeManager fs = VolumeManagerImpl.get();
+
+ Matcher rowMatcher = null;
+ KeyExtent ke = null;
+ Text row = null;
+ if (opts.files.isEmpty()) {
+ new JCommander(opts).usage();
+ return;
+ }
+ if (opts.row != null)
+ row = new Text(opts.row);
+ if (opts.extent != null) {
+ String sa[] = opts.extent.split(";");
+ ke = new KeyExtent(new Text(sa[0]), new Text(sa[1]), new Text(sa[2]));
+ }
+ if (opts.regexp != null) {
+ Pattern pattern = Pattern.compile(opts.regexp);
+ rowMatcher = pattern.matcher("");
+ }
+
+ Set<Integer> tabletIds = new HashSet<Integer>();
+
+ for (String file : opts.files) {
+
+ Path path = new Path(file);
+ LogFileKey key = new LogFileKey();
+ LogFileValue value = new LogFileValue();
+
+ if (fs.isFile(path)) {
+ // read log entries from a simple hdfs file
+ DFSLoggerInputStreams streams = DfsLogger.readHeaderAndReturnStream(fs, path, ServerConfiguration.getSiteConfiguration());
+ DataInputStream input = streams.getDecryptingInputStream();
+
+ try {
+ while (true) {
+ try {
+ key.readFields(input);
+ value.readFields(input);
+ } catch (EOFException ex) {
+ break;
+ }
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
+ }
+ } finally {
+ input.close();
+ }
+ } else {
+ // read the log entries sorted in a map file
+ MultiReader input = new MultiReader(fs, path);
+ while (input.next(key, value)) {
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
+ }
+ }
+ }
+ }
+
+ public static void printLogEvent(LogFileKey key, LogFileValue value, Text row, Matcher rowMatcher, KeyExtent ke, Set<Integer> tabletIds, int maxMutations) {
+
+ if (ke != null) {
+ if (key.event == LogEvents.DEFINE_TABLET) {
+ if (key.tablet.equals(ke)) {
+ tabletIds.add(key.tid);
+ } else {
+ return;
+ }
+ } else if (!tabletIds.contains(key.tid)) {
+ return;
+ }
+ }
+
+ if (row != null || rowMatcher != null) {
+ if (key.event == LogEvents.MUTATION || key.event == LogEvents.MANY_MUTATIONS) {
+ boolean found = false;
+ for (Mutation m : value.mutations) {
+ if (row != null && new Text(m.getRow()).equals(row)) {
+ found = true;
+ break;
+ }
+
+ if (rowMatcher != null) {
+ rowMatcher.reset(new String(m.getRow(), Constants.UTF8));
+ if (rowMatcher.matches()) {
+ found = true;
+ break;
+ }
+ }
+ }
+
+ if (!found)
+ return;
+ } else {
+ return;
+ }
+
+ }
+
+ System.out.println(key);
+ System.out.println(LogFileValue.format(value, maxMutations));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
----------------------------------------------------------------------
diff --cc start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
index b7d5fa9,277c741..745401b
--- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
+++ b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
@@@ -36,15 -30,16 +30,16 @@@ public class PostDelegatingVFSClassLoad
super(files, manager, parent);
}
+ @Override
protected synchronized Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
Class<?> c = findLoadedClass(name);
- if (c == null) {
- try {
- // try finding this class here instead of parent
- findClass(name);
- } catch (ClassNotFoundException e) {
-
- }
+ if (c != null)
+ return c;
+ try {
+ // try finding this class here instead of parent
+ return findClass(name);
+ } catch (ClassNotFoundException e) {
+
}
return super.loadClass(name, resolve);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java
----------------------------------------------------------------------
diff --cc start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java
index c9fd2f5,104ea09..2973750
--- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java
+++ b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java
@@@ -42,76 -42,36 +42,71 @@@ import org.apache.hadoop.fs.Path
*/
public class HdfsFileSystem extends AbstractFileSystem
{
- private static final Log log = LogFactory.getLog(HdfsFileSystem.class);
-
- private FileSystem fs;
-
- protected HdfsFileSystem(final FileName rootName, final FileSystemOptions fileSystemOptions)
+ private static final Log log = LogFactory.getLog(HdfsFileSystem.class);
+
+ private FileSystem fs;
+
- /**
- *
- * @param rootName
- * @param fileSystemOptions
- */
+ protected HdfsFileSystem(final FileName rootName, final FileSystemOptions fileSystemOptions)
+ {
+ super(rootName, null, fileSystemOptions);
+ }
+
+ /**
+ * @see org.apache.commons.vfs2.provider.AbstractFileSystem#addCapabilities(java.util.Collection)
+ */
+ @Override
+ protected void addCapabilities(final Collection<Capability> capabilities)
+ {
+ capabilities.addAll(HdfsFileProvider.CAPABILITIES);
+ }
+
+ /**
+ * @see org.apache.commons.vfs2.provider.AbstractFileSystem#close()
+ */
+ @Override
+ synchronized public void close()
+ {
+ try
{
- super(rootName, null, fileSystemOptions);
+ if (null != fs)
+ {
+ fs.close();
+ }
}
-
- /**
- * @see org.apache.commons.vfs2.provider.AbstractFileSystem#addCapabilities(java.util.Collection)
- */
- @Override
- protected void addCapabilities(final Collection<Capability> capabilities)
+ catch (final IOException e)
{
- capabilities.addAll(HdfsFileProvider.CAPABILITIES);
+ throw new RuntimeException("Error closing HDFS client", e);
}
-
- /**
- * @see org.apache.commons.vfs2.provider.AbstractFileSystem#close()
- */
- @Override
- public void close()
+ super.close();
+ }
+
+ /**
+ * @see org.apache.commons.vfs2.provider.AbstractFileSystem#createFile(org.apache.commons.vfs2.provider.AbstractFileName)
+ */
+ @Override
+ protected FileObject createFile(final AbstractFileName name) throws Exception
+ {
+ throw new FileSystemException("Operation not supported");
+ }
+
+ /**
+ * @see org.apache.commons.vfs2.provider.AbstractFileSystem#resolveFile(org.apache.commons.vfs2.FileName)
+ */
+ @Override
+ public FileObject resolveFile(final FileName name) throws FileSystemException
+ {
+
+ synchronized (this)
{
+ if (null == this.fs)
+ {
+ final String hdfsUri = name.getRootURI();
+ final Configuration conf = new Configuration(true);
+ conf.set(org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY, hdfsUri);
+ this.fs = null;
try
{
- if (null != fs)
- {
- fs.close();
- }
+ fs = org.apache.hadoop.fs.FileSystem.get(conf);
}
catch (final IOException e)
{
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/test/src/main/java/org/apache/accumulo/test/functional/CacheTestClean.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
----------------------------------------------------------------------
[45/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/data/Range.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Range.java
index 65873c3,0000000..122436b
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/data/Range.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Range.java
@@@ -1,906 -1,0 +1,899 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
+import java.io.DataInput;
+import java.io.DataOutput;
- import java.io.InvalidObjectException;
+import java.io.IOException;
++import java.io.InvalidObjectException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.accumulo.core.data.thrift.TRange;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+
+/**
+ * This class is used to specify a range of Accumulo Keys.
+ *
+ */
+
+public class Range implements WritableComparable<Range> {
+
+ private Key start;
+ private Key stop;
+ private boolean startKeyInclusive;
+ private boolean stopKeyInclusive;
+ private boolean infiniteStartKey;
+ private boolean infiniteStopKey;
+
+ /**
+ * Creates a range that goes from negative to positive infinity
+ */
+
+ public Range() {
+ this((Key) null, true, (Key) null, true);
+ }
+
+ /**
+ * Creates a range from startKey inclusive to endKey inclusive
+ *
+ * @param startKey
+ * set this to null when negative infinity is needed
+ * @param endKey
+ * set this to null when positive infinity is needed
+ */
+ public Range(Key startKey, Key endKey) {
+ this(startKey, true, endKey, true);
+ }
+
+ /**
+ * Creates a range that covers an entire row
+ *
+ * @param row
+ * set this to null to cover all rows
+ */
+ public Range(CharSequence row) {
+ this(row, true, row, true);
+ }
+
+ /**
+ * Creates a range that covers an entire row
+ *
+ * @param row
+ * set this to null to cover all rows
+ */
+ public Range(Text row) {
+ this(row, true, row, true);
+ }
+
+ /**
+ * Creates a range from startRow inclusive to endRow inclusive
+ *
+ * @param startRow
+ * set this to null when negative infinity is needed
+ * @param endRow
+ * set this to null when positive infinity is needed
+ */
+ public Range(Text startRow, Text endRow) {
+ this(startRow, true, endRow, true);
+ }
+
+ /**
+ * Creates a range from startRow inclusive to endRow inclusive
+ *
+ * @param startRow
+ * set this to null when negative infinity is needed
+ * @param endRow
+ * set this to null when positive infinity is needed
+ */
+ public Range(CharSequence startRow, CharSequence endRow) {
+ this(startRow, true, endRow, true);
+ }
+
+ /**
+ * Creates a range from startRow to endRow
+ *
+ * @param startRow
+ * set this to null when negative infinity is needed
+ * @param startRowInclusive
+ * determines if the start row is skipped
+ * @param endRow
+ * set this to null when positive infinity is needed
+ * @param endRowInclusive
+ * determines if the endRow is included
+ */
+
+ public Range(Text startRow, boolean startRowInclusive, Text endRow, boolean endRowInclusive) {
+ this((startRow == null ? null : (startRowInclusive ? new Key(startRow) : new Key(startRow).followingKey(PartialKey.ROW))), true, (endRow == null ? null
+ : (endRowInclusive ? new Key(endRow).followingKey(PartialKey.ROW) : new Key(endRow))), false);
+ }
+
+ /**
+ * Creates a range from startRow to endRow
+ *
+ * @param startRow
+ * set this to null when negative infinity is needed
+ * @param startRowInclusive
+ * determines if the start row is skipped
+ * @param endRow
+ * set this to null when positive infinity is needed
+ * @param endRowInclusive
+ * determines if the endRow is included
+ */
+
+ public Range(CharSequence startRow, boolean startRowInclusive, CharSequence endRow, boolean endRowInclusive) {
+ this(startRow == null ? null : new Text(startRow.toString()), startRowInclusive, endRow == null ? null : new Text(endRow.toString()), endRowInclusive);
+ }
+
+ /**
+ * Creates a range from startKey to endKey
+ *
+ * @param startKey
+ * set this to null when negative infinity is needed
+ * @param startKeyInclusive
+ * determines if the ranges includes the start key
+ * @param endKey
+ * set this to null when infinity is needed
+ * @param endKeyInclusive
+ * determines if the range includes the end key
+ */
+ public Range(Key startKey, boolean startKeyInclusive, Key endKey, boolean endKeyInclusive) {
+ this.start = startKey;
+ this.startKeyInclusive = startKeyInclusive;
+ this.infiniteStartKey = startKey == null;
+ this.stop = endKey;
+ this.stopKeyInclusive = endKeyInclusive;
+ this.infiniteStopKey = stop == null;
+
+ if (!infiniteStartKey && !infiniteStopKey && beforeStartKey(endKey)) {
+ throw new IllegalArgumentException("Start key must be less than end key in range (" + startKey + ", " + endKey + ")");
+ }
+ }
+
+ /**
+ * Copies a range
+ */
+ public Range(Range range) {
+ this(range.start, range.startKeyInclusive, range.infiniteStartKey, range.stop, range.stopKeyInclusive, range.infiniteStopKey);
+ }
+
+ /**
+ * Creates a range from start to stop.
+ *
+ * @param start
+ * set this to null when negative infinity is needed
+ * @param stop
+ * set this to null when infinity is needed
+ * @param startKeyInclusive
+ * determines if the ranges includes the start key
+ * @param stopKeyInclusive
+ * determines if the range includes the end key
+ * @param infiniteStartKey
+ * true if start key is negative infinity (null)
+ * @param infiniteStopKey
+ * true if stop key is positive infinity (null)
+ * @throws IllegalArgumentException if stop is before start, or infiniteStartKey is true but start is not null, or infiniteStopKey is true but stop is not
+ * null
+ */
+ public Range(Key start, Key stop, boolean startKeyInclusive, boolean stopKeyInclusive, boolean infiniteStartKey, boolean infiniteStopKey) {
+ this(start, startKeyInclusive, infiniteStartKey, stop, stopKeyInclusive, infiniteStopKey);
+ if (!infiniteStartKey && !infiniteStopKey && beforeStartKey(stop)) {
+ throw new IllegalArgumentException("Start key must be less than end key in range (" + start + ", " + stop + ")");
+ }
+ }
+
+ /**
+ * Creates a range from start to stop. Unlike the public six-argument method,
+ * this one does not assure that stop is after start, which helps performance
+ * in cases where that assurance is already in place.
+ *
+ * @param start
+ * set this to null when negative infinity is needed
+ * @param startKeyInclusive
+ * determines if the ranges includes the start key
+ * @param infiniteStartKey
+ * true if start key is negative infinity (null)
+ * @param stop
+ * set this to null when infinity is needed
+ * @param stopKeyInclusive
+ * determines if the range includes the end key
+ * @param infiniteStopKey
+ * true if stop key is positive infinity (null)
+ * @throws IllegalArgumentException if infiniteStartKey is true but start is not null, or infiniteStopKey is true but stop is not null
+ */
+ protected Range(Key start, boolean startKeyInclusive, boolean infiniteStartKey, Key stop, boolean stopKeyInclusive, boolean infiniteStopKey) {
+ if (infiniteStartKey && start != null)
+ throw new IllegalArgumentException();
+
+ if (infiniteStopKey && stop != null)
+ throw new IllegalArgumentException();
+
+ this.start = start;
+ this.stop = stop;
+ this.startKeyInclusive = startKeyInclusive;
+ this.stopKeyInclusive = stopKeyInclusive;
+ this.infiniteStartKey = infiniteStartKey;
+ this.infiniteStopKey = infiniteStopKey;
+ }
+
+ public Range(TRange trange) {
+ this(trange.start == null ? null : new Key(trange.start), trange.startKeyInclusive, trange.infiniteStartKey,
+ trange.stop == null ? null : new Key(trange.stop), trange.stopKeyInclusive, trange.infiniteStopKey);
+ if (!infiniteStartKey && !infiniteStopKey && beforeStartKey(stop)) {
+ throw new IllegalArgumentException("Start key must be less than end key in range (" + start + ", " + stop + ")");
+ }
+ }
+
+ /**
+ * @return the first key in the range, null if the key is infinite
+ */
+ public Key getStartKey() {
+ if (infiniteStartKey) {
+ return null;
+ }
+ return start;
+ }
+
+ /**
+ *
- * @param key
+ * @return true if the given key is before the range, otherwise false
+ */
+ public boolean beforeStartKey(Key key) {
+ if (infiniteStartKey) {
+ return false;
+ }
+
+ if (startKeyInclusive)
+ return key.compareTo(start) < 0;
+ return key.compareTo(start) <= 0;
+ }
+
+ /**
+ * @return the last key in the range, null if the end key is infinite
+ */
+
+ public Key getEndKey() {
+ if (infiniteStopKey) {
+ return null;
+ }
+ return stop;
+ }
+
+ /**
- * @param key
+ * @return true if the given key is after the range, otherwise false
+ */
+
+ public boolean afterEndKey(Key key) {
+ if (infiniteStopKey)
+ return false;
+
+ if (stopKeyInclusive)
+ return stop.compareTo(key) < 0;
+ return stop.compareTo(key) <= 0;
+ }
+
+ @Override
+ public int hashCode() {
+ int startHash = infiniteStartKey ? 0 : start.hashCode() + (startKeyInclusive ? 1 : 0);
+ int stopHash = infiniteStopKey ? 0 : stop.hashCode() + (stopKeyInclusive ? 1 : 0);
+
+ return startHash + stopHash;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof Range)
+ return equals((Range) o);
+ return false;
+ }
+
+ public boolean equals(Range otherRange) {
+
+ return compareTo(otherRange) == 0;
+ }
+
+ /**
+ * Compares this range to another range. Compares in the order start key, inclusiveness of start key, end key, inclusiveness of end key. Infinite keys sort
+ * first, and non-infinite keys are compared with {@link Key#compareTo(Key)}. Inclusive sorts before non-inclusive.
+ */
++ @Override
+ public int compareTo(Range o) {
+ int comp;
+
+ if (infiniteStartKey)
+ if (o.infiniteStartKey)
+ comp = 0;
+ else
+ comp = -1;
+ else if (o.infiniteStartKey)
+ comp = 1;
+ else {
+ comp = start.compareTo(o.start);
+ if (comp == 0)
+ if (startKeyInclusive && !o.startKeyInclusive)
+ comp = -1;
+ else if (!startKeyInclusive && o.startKeyInclusive)
+ comp = 1;
+
+ }
+
+ if (comp == 0)
+ if (infiniteStopKey)
+ if (o.infiniteStopKey)
+ comp = 0;
+ else
+ comp = 1;
+ else if (o.infiniteStopKey)
+ comp = -1;
+ else {
+ comp = stop.compareTo(o.stop);
+ if (comp == 0)
+ if (stopKeyInclusive && !o.stopKeyInclusive)
+ comp = 1;
+ else if (!stopKeyInclusive && o.stopKeyInclusive)
+ comp = -1;
+ }
+
+ return comp;
+ }
+
+ /**
+ *
- * @param key
+ * @return true if the given key falls within the range
+ */
+ public boolean contains(Key key) {
+ return !beforeStartKey(key) && !afterEndKey(key);
+ }
+
+ /**
+ * Takes a collection on range and merges overlapping and adjacent ranges. For example given the following input
+ *
+ * <pre>
+ * [a,c], (c, d], (g,m), (j,t]
+ * </pre>
+ *
+ * the following ranges would be returned
+ *
+ * <pre>
+ * [a,d], (g,t]
+ * </pre>
+ *
- * @param ranges
+ * @return list of merged ranges
+ */
+
+ public static List<Range> mergeOverlapping(Collection<Range> ranges) {
+ if (ranges.size() == 0)
+ return Collections.emptyList();
+
+ List<Range> ral = new ArrayList<Range>(ranges);
+ Collections.sort(ral);
+
+ ArrayList<Range> ret = new ArrayList<Range>(ranges.size());
+
+ Range currentRange = ral.get(0);
+ boolean currentStartKeyInclusive = ral.get(0).startKeyInclusive;
+
+ for (int i = 1; i < ral.size(); i++) {
+ // because of inclusive switch, equal keys may not be seen
+
+ if (currentRange.infiniteStopKey) {
+ // this range has the minimal start key and
+ // an infinite end key so it will contain all
+ // other ranges
+ break;
+ }
+
+ Range range = ral.get(i);
+
+ boolean startKeysEqual;
+ if (range.infiniteStartKey) {
+ // previous start key must be infinite because it is sorted
+ assert (currentRange.infiniteStartKey);
+ startKeysEqual = true;
+ } else if (currentRange.infiniteStartKey) {
+ startKeysEqual = false;
+ } else if (currentRange.start.equals(range.start)) {
+ startKeysEqual = true;
+ } else {
+ startKeysEqual = false;
+ }
+
+ if (startKeysEqual || currentRange.contains(range.start)
+ || (!currentRange.stopKeyInclusive && range.startKeyInclusive && range.start.equals(currentRange.stop))) {
+ int cmp;
+
+ if (range.infiniteStopKey || (cmp = range.stop.compareTo(currentRange.stop)) > 0 || (cmp == 0 && range.stopKeyInclusive)) {
+ currentRange = new Range(currentRange.getStartKey(), currentStartKeyInclusive, range.getEndKey(), range.stopKeyInclusive);
+ }/* else currentRange contains ral.get(i) */
+ } else {
+ ret.add(currentRange);
+ currentRange = range;
+ currentStartKeyInclusive = range.startKeyInclusive;
+ }
+ }
+
+ ret.add(currentRange);
+
+ return ret;
+ }
+
+ /**
+ * Creates a range which represents the intersection of this range and the passed in range. The following example will print true.
+ *
+ * <pre>
+ * Range range1 = new Range("a", "f");
+ * Range range2 = new Range("c", "n");
+ * Range range3 = range1.clip(range2);
+ * System.out.println(range3.equals(new Range("c", "f")));
+ * </pre>
+ *
- * @param range
+ * @return the intersection
+ * @throws IllegalArgumentException
+ * if ranges does not overlap
+ */
+
+ public Range clip(Range range) {
+ return clip(range, false);
+ }
+
+ /**
+ * Same as other clip function except if gives the option to return null of the ranges do not overlap instead of throwing an exception.
+ *
+ * @see Range#clip(Range)
- * @param range
+ * @param returnNullIfDisjoint
+ * If the ranges do not overlap and true is passed, then null is returned otherwise an exception is thrown.
+ * @return the intersection
+ */
+
+ public Range clip(Range range, boolean returnNullIfDisjoint) {
+
+ Key sk = range.getStartKey();
+ boolean ski = range.isStartKeyInclusive();
+
+ Key ek = range.getEndKey();
+ boolean eki = range.isEndKeyInclusive();
+
+ if (range.getStartKey() == null) {
+ if (getStartKey() != null) {
+ sk = getStartKey();
+ ski = isStartKeyInclusive();
+ }
+ } else if (afterEndKey(range.getStartKey())
+ || (getEndKey() != null && range.getStartKey().equals(getEndKey()) && !(range.isStartKeyInclusive() && isEndKeyInclusive()))) {
+ if (returnNullIfDisjoint)
+ return null;
+ throw new IllegalArgumentException("Range " + range + " does not overlap " + this);
+ } else if (beforeStartKey(range.getStartKey())) {
+ sk = getStartKey();
+ ski = isStartKeyInclusive();
+ }
+
+ if (range.getEndKey() == null) {
+ if (getEndKey() != null) {
+ ek = getEndKey();
+ eki = isEndKeyInclusive();
+ }
+ } else if (beforeStartKey(range.getEndKey())
+ || (getStartKey() != null && range.getEndKey().equals(getStartKey()) && !(range.isEndKeyInclusive() && isStartKeyInclusive()))) {
+ if (returnNullIfDisjoint)
+ return null;
+ throw new IllegalArgumentException("Range " + range + " does not overlap " + this);
+ } else if (afterEndKey(range.getEndKey())) {
+ ek = getEndKey();
+ eki = isEndKeyInclusive();
+ }
+
+ return new Range(sk, ski, ek, eki);
+ }
+
+ /**
+ * Creates a new range that is bounded by the columns passed in. The stary key in the returned range will have a column >= to the minimum column. The end key
+ * in the returned range will have a column <= the max column.
+ *
- *
- * @param min
- * @param max
+ * @return a column bounded range
+ * @throws IllegalArgumentException
+ * if min > max
+ */
+
+ public Range bound(Column min, Column max) {
+
+ if (min.compareTo(max) > 0) {
+ throw new IllegalArgumentException("min column > max column " + min + " " + max);
+ }
+
+ Key sk = getStartKey();
+ boolean ski = isStartKeyInclusive();
+
+ if (sk != null) {
+
+ ByteSequence cf = sk.getColumnFamilyData();
+ ByteSequence cq = sk.getColumnQualifierData();
+
+ ByteSequence mincf = new ArrayByteSequence(min.columnFamily);
+ ByteSequence mincq;
+
+ if (min.columnQualifier != null)
+ mincq = new ArrayByteSequence(min.columnQualifier);
+ else
+ mincq = new ArrayByteSequence(new byte[0]);
+
+ int cmp = cf.compareTo(mincf);
+
+ if (cmp < 0 || (cmp == 0 && cq.compareTo(mincq) < 0)) {
+ ski = true;
+ sk = new Key(sk.getRowData().toArray(), mincf.toArray(), mincq.toArray(), new byte[0], Long.MAX_VALUE, true);
+ }
+ }
+
+ Key ek = getEndKey();
+ boolean eki = isEndKeyInclusive();
+
+ if (ek != null) {
+ ByteSequence row = ek.getRowData();
+ ByteSequence cf = ek.getColumnFamilyData();
+ ByteSequence cq = ek.getColumnQualifierData();
+ ByteSequence cv = ek.getColumnVisibilityData();
+
+ ByteSequence maxcf = new ArrayByteSequence(max.columnFamily);
+ ByteSequence maxcq = null;
+ if (max.columnQualifier != null)
+ maxcq = new ArrayByteSequence(max.columnQualifier);
+
+ boolean set = false;
+
+ int comp = cf.compareTo(maxcf);
+
+ if (comp > 0) {
+ set = true;
+ } else if (comp == 0 && maxcq != null && cq.compareTo(maxcq) > 0) {
+ set = true;
+ } else if (!eki && row.length() > 0 && row.byteAt(row.length() - 1) == 0 && cf.length() == 0 && cq.length() == 0 && cv.length() == 0
+ && ek.getTimestamp() == Long.MAX_VALUE) {
+ row = row.subSequence(0, row.length() - 1);
+ set = true;
+ }
+
+ if (set) {
+ eki = false;
+ if (maxcq == null)
+ ek = new Key(row.toArray(), maxcf.toArray(), new byte[0], new byte[0], 0, false).followingKey(PartialKey.ROW_COLFAM);
+ else
+ ek = new Key(row.toArray(), maxcf.toArray(), maxcq.toArray(), new byte[0], 0, false).followingKey(PartialKey.ROW_COLFAM_COLQUAL);
+ }
+ }
+
+ return new Range(sk, ski, ek, eki);
+ }
+
++ @Override
+ public String toString() {
+ return ((startKeyInclusive && start != null) ? "[" : "(") + (start == null ? "-inf" : start) + "," + (stop == null ? "+inf" : stop)
+ + ((stopKeyInclusive && stop != null) ? "]" : ")");
+ }
+
++ @Override
+ public void readFields(DataInput in) throws IOException {
+ infiniteStartKey = in.readBoolean();
+ infiniteStopKey = in.readBoolean();
+ if (!infiniteStartKey) {
+ start = new Key();
+ start.readFields(in);
+ } else {
+ start = null;
+ }
+
+ if (!infiniteStopKey) {
+ stop = new Key();
+ stop.readFields(in);
+ } else {
+ stop = null;
+ }
+
+ startKeyInclusive = in.readBoolean();
+ stopKeyInclusive = in.readBoolean();
+
+ if (!infiniteStartKey && !infiniteStopKey && beforeStartKey(stop)) {
+ throw new InvalidObjectException("Start key must be less than end key in range (" + start + ", " + stop + ")");
+ }
+ }
+
++ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeBoolean(infiniteStartKey);
+ out.writeBoolean(infiniteStopKey);
+ if (!infiniteStartKey)
+ start.write(out);
+ if (!infiniteStopKey)
+ stop.write(out);
+ out.writeBoolean(startKeyInclusive);
+ out.writeBoolean(stopKeyInclusive);
+ }
+
+ public boolean isStartKeyInclusive() {
+ return startKeyInclusive;
+ }
+
+ public boolean isEndKeyInclusive() {
+ return stopKeyInclusive;
+ }
+
+ public TRange toThrift() {
+ return new TRange(start == null ? null : start.toThrift(), stop == null ? null : stop.toThrift(), startKeyInclusive, stopKeyInclusive, infiniteStartKey,
+ infiniteStopKey);
+ }
+
+ public boolean isInfiniteStartKey() {
+ return infiniteStartKey;
+ }
+
+ public boolean isInfiniteStopKey() {
+ return infiniteStopKey;
+ }
+
+ /**
+ * Creates a range that covers an exact row Returns the same Range as new Range(row)
+ *
+ * @param row
+ * all keys in the range will have this row
+ */
+ public static Range exact(Text row) {
+ return new Range(row);
+ }
+
+ /**
+ * Creates a range that covers an exact row and column family
+ *
+ * @param row
+ * all keys in the range will have this row
+ *
+ * @param cf
+ * all keys in the range will have this column family
+ */
+ public static Range exact(Text row, Text cf) {
+ Key startKey = new Key(row, cf);
+ return new Range(startKey, true, startKey.followingKey(PartialKey.ROW_COLFAM), false);
+ }
+
+ /**
+ * Creates a range that covers an exact row, column family, and column qualifier
+ *
+ * @param row
+ * all keys in the range will have this row
+ *
+ * @param cf
+ * all keys in the range will have this column family
+ *
+ * @param cq
+ * all keys in the range will have this column qualifier
+ */
+ public static Range exact(Text row, Text cf, Text cq) {
+ Key startKey = new Key(row, cf, cq);
+ return new Range(startKey, true, startKey.followingKey(PartialKey.ROW_COLFAM_COLQUAL), false);
+ }
+
+ /**
+ * Creates a range that covers an exact row, column family, column qualifier, and visibility
+ *
+ * @param row
+ * all keys in the range will have this row
+ *
+ * @param cf
+ * all keys in the range will have this column family
+ *
+ * @param cq
+ * all keys in the range will have this column qualifier
+ *
+ * @param cv
+ * all keys in the range will have this column visibility
+ */
+ public static Range exact(Text row, Text cf, Text cq, Text cv) {
+ Key startKey = new Key(row, cf, cq, cv);
+ return new Range(startKey, true, startKey.followingKey(PartialKey.ROW_COLFAM_COLQUAL_COLVIS), false);
+ }
+
+ /**
+ * Creates a range that covers an exact row, column family, column qualifier, visibility, and timestamp
+ *
+ * @param row
+ * all keys in the range will have this row
+ *
+ * @param cf
+ * all keys in the range will have this column family
+ *
+ * @param cq
+ * all keys in the range will have this column qualifier
+ *
+ * @param cv
+ * all keys in the range will have this column visibility
+ *
+ * @param ts
+ * all keys in the range will have this timestamp
+ */
+ public static Range exact(Text row, Text cf, Text cq, Text cv, long ts) {
+ Key startKey = new Key(row, cf, cq, cv, ts);
+ return new Range(startKey, true, startKey.followingKey(PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME), false);
+ }
+
+ /**
+ * Returns a Text that sorts just after all Texts beginning with a prefix
- *
- * @param prefix
+ */
+ public static Text followingPrefix(Text prefix) {
+ byte[] prefixBytes = prefix.getBytes();
+
+ // find the last byte in the array that is not 0xff
+ int changeIndex = prefix.getLength() - 1;
+ while (changeIndex >= 0 && prefixBytes[changeIndex] == (byte) 0xff)
+ changeIndex--;
+ if (changeIndex < 0)
+ return null;
+
+ // copy prefix bytes into new array
+ byte[] newBytes = new byte[changeIndex + 1];
+ System.arraycopy(prefixBytes, 0, newBytes, 0, changeIndex + 1);
+
+ // increment the selected byte
+ newBytes[changeIndex]++;
+ return new Text(newBytes);
+ }
+
+ /**
+ * Returns a Range that covers all rows beginning with a prefix
+ *
+ * @param rowPrefix
+ * all keys in the range will have rows that begin with this prefix
+ */
+ public static Range prefix(Text rowPrefix) {
+ Text fp = followingPrefix(rowPrefix);
+ return new Range(new Key(rowPrefix), true, fp == null ? null : new Key(fp), false);
+ }
+
+ /**
+ * Returns a Range that covers all column families beginning with a prefix within a given row
+ *
+ * @param row
+ * all keys in the range will have this row
+ *
+ * @param cfPrefix
+ * all keys in the range will have column families that begin with this prefix
+ */
+ public static Range prefix(Text row, Text cfPrefix) {
+ Text fp = followingPrefix(cfPrefix);
+ return new Range(new Key(row, cfPrefix), true, fp == null ? new Key(row).followingKey(PartialKey.ROW) : new Key(row, fp), false);
+ }
+
+ /**
+ * Returns a Range that covers all column qualifiers beginning with a prefix within a given row and column family
+ *
+ * @param row
+ * all keys in the range will have this row
+ *
+ * @param cf
+ * all keys in the range will have this column family
+ *
+ * @param cqPrefix
+ * all keys in the range will have column qualifiers that begin with this prefix
+ */
+ public static Range prefix(Text row, Text cf, Text cqPrefix) {
+ Text fp = followingPrefix(cqPrefix);
+ return new Range(new Key(row, cf, cqPrefix), true, fp == null ? new Key(row, cf).followingKey(PartialKey.ROW_COLFAM) : new Key(row, cf, fp), false);
+ }
+
+ /**
+ * Returns a Range that covers all column visibilities beginning with a prefix within a given row, column family, and column qualifier
+ *
+ * @param row
+ * all keys in the range will have this row
+ *
+ * @param cf
+ * all keys in the range will have this column family
+ *
+ * @param cq
+ * all keys in the range will have this column qualifier
+ *
+ * @param cvPrefix
+ * all keys in the range will have column visibilities that begin with this prefix
+ */
+ public static Range prefix(Text row, Text cf, Text cq, Text cvPrefix) {
+ Text fp = followingPrefix(cvPrefix);
+ return new Range(new Key(row, cf, cq, cvPrefix), true, fp == null ? new Key(row, cf, cq).followingKey(PartialKey.ROW_COLFAM_COLQUAL) : new Key(row, cf, cq,
+ fp), false);
+ }
+
+ /**
+ * Creates a range that covers an exact row
+ *
+ * @see Range#exact(Text)
+ */
+ public static Range exact(CharSequence row) {
+ return Range.exact(new Text(row.toString()));
+ }
+
+ /**
+ * Creates a range that covers an exact row and column family
+ *
+ * @see Range#exact(Text, Text)
+ */
+ public static Range exact(CharSequence row, CharSequence cf) {
+ return Range.exact(new Text(row.toString()), new Text(cf.toString()));
+ }
+
+ /**
+ * Creates a range that covers an exact row, column family, and column qualifier
+ *
+ * @see Range#exact(Text, Text, Text)
+ */
+ public static Range exact(CharSequence row, CharSequence cf, CharSequence cq) {
+ return Range.exact(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()));
+ }
+
+ /**
+ * Creates a range that covers an exact row, column family, column qualifier, and visibility
+ *
+ * @see Range#exact(Text, Text, Text, Text)
+ */
+ public static Range exact(CharSequence row, CharSequence cf, CharSequence cq, CharSequence cv) {
+ return Range.exact(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()), new Text(cv.toString()));
+ }
+
+ /**
+ * Creates a range that covers an exact row, column family, column qualifier, visibility, and timestamp
+ *
+ * @see Range#exact(Text, Text, Text, Text, long)
+ */
+ public static Range exact(CharSequence row, CharSequence cf, CharSequence cq, CharSequence cv, long ts) {
+ return Range.exact(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()), new Text(cv.toString()), ts);
+ }
+
+ /**
+ * Returns a Range that covers all rows beginning with a prefix
+ *
+ * @see Range#prefix(Text)
+ */
+ public static Range prefix(CharSequence rowPrefix) {
+ return Range.prefix(new Text(rowPrefix.toString()));
+ }
+
+ /**
+ * Returns a Range that covers all column families beginning with a prefix within a given row
+ *
+ * @see Range#prefix(Text, Text)
+ */
+ public static Range prefix(CharSequence row, CharSequence cfPrefix) {
+ return Range.prefix(new Text(row.toString()), new Text(cfPrefix.toString()));
+ }
+
+ /**
+ * Returns a Range that covers all column qualifiers beginning with a prefix within a given row and column family
+ *
+ * @see Range#prefix(Text, Text, Text)
+ */
+ public static Range prefix(CharSequence row, CharSequence cf, CharSequence cqPrefix) {
+ return Range.prefix(new Text(row.toString()), new Text(cf.toString()), new Text(cqPrefix.toString()));
+ }
+
+ /**
+ * Returns a Range that covers all column visibilities beginning with a prefix within a given row, column family, and column qualifier
+ *
+ * @see Range#prefix(Text, Text, Text, Text)
+ */
+ public static Range prefix(CharSequence row, CharSequence cf, CharSequence cq, CharSequence cvPrefix) {
+ return Range.prefix(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()), new Text(cvPrefix.toString()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java
index 2b3cdf5,0000000..0ac5308
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java
@@@ -1,181 -1,0 +1,176 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.file.rfile;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.blockfile.ABlockReader;
+import org.apache.accumulo.core.file.rfile.MultiLevelIndex.IndexEntry;
+
+/**
+ *
+ */
+public class BlockIndex {
+
+ public static BlockIndex getIndex(ABlockReader cacheBlock, IndexEntry indexEntry) throws IOException {
+
+ BlockIndex blockIndex = cacheBlock.getIndex(BlockIndex.class);
+
+ int accessCount = blockIndex.accessCount.incrementAndGet();
+
+ // 1 is a power of two, but do not care about it
+ if (accessCount >= 2 && isPowerOfTwo(accessCount)) {
+ blockIndex.buildIndex(accessCount, cacheBlock, indexEntry);
+ }
+
+ if (blockIndex.blockIndex != null)
+ return blockIndex;
+
+ return null;
+ }
+
+ private static boolean isPowerOfTwo(int x) {
+ return ((x > 0) && (x & (x - 1)) == 0);
+ }
+
+ private AtomicInteger accessCount = new AtomicInteger(0);
+ private volatile BlockIndexEntry[] blockIndex = null;
+
+ public static class BlockIndexEntry implements Comparable<BlockIndexEntry> {
+
+ private Key prevKey;
+ private int entriesLeft;
+ private int pos;
+
+ public BlockIndexEntry(int pos, int entriesLeft, Key prevKey) {
+ this.pos = pos;
+ this.entriesLeft = entriesLeft;
+ this.prevKey = prevKey;
+ }
+
- /**
- * @param key
- */
+ public BlockIndexEntry(Key key) {
+ this.prevKey = key;
+ }
-
-
+
+ public int getEntriesLeft() {
+ return entriesLeft;
+ }
+
+ @Override
+ public int compareTo(BlockIndexEntry o) {
+ return prevKey.compareTo(o.prevKey);
+ }
+
+ @Override
+ public String toString() {
+ return prevKey + " " + entriesLeft + " " + pos;
+ }
+
+ public Key getPrevKey() {
+ return prevKey;
+ }
+ }
+
+ public BlockIndexEntry seekBlock(Key startKey, ABlockReader cacheBlock) {
+
+ // get a local ref to the index, another thread could change it
+ BlockIndexEntry[] blockIndex = this.blockIndex;
+
+ int pos = Arrays.binarySearch(blockIndex, new BlockIndexEntry(startKey));
+
+ int index;
+
+ if (pos < 0) {
+ if (pos == -1)
+ return null; // less than the first key in index, did not index the first key in block so just return null... code calling this will scan from beginning
+ // of block
+ index = (pos * -1) - 2;
+ } else {
+ // found exact key in index
+ index = pos;
+ while (index > 0) {
+ if (blockIndex[index].getPrevKey().equals(startKey))
+ index--;
+ else
+ break;
+ }
+ }
+
+ // handle case where multiple keys in block are exactly the same, want to find the earliest key in the index
+ while (index - 1 > 0) {
+ if (blockIndex[index].getPrevKey().equals(blockIndex[index - 1].getPrevKey()))
+ index--;
+ else
+ break;
+
+ }
+
+ if (index == 0 && blockIndex[index].getPrevKey().equals(startKey))
+ return null;
+
+ BlockIndexEntry bie = blockIndex[index];
+ cacheBlock.seek(bie.pos);
+ return bie;
+ }
+
+ private synchronized void buildIndex(int indexEntries, ABlockReader cacheBlock, IndexEntry indexEntry) throws IOException {
+ cacheBlock.seek(0);
+
+ RelativeKey rk = new RelativeKey();
+ Value val = new Value();
+
+ int interval = indexEntry.getNumEntries() / indexEntries;
+
+ if (interval <= 32)
+ return;
+
+ // multiple threads could try to create the index with different sizes, do not replace a large index with a smaller one
+ if (this.blockIndex != null && this.blockIndex.length > indexEntries - 1)
+ return;
+
+ int count = 0;
+
+ ArrayList<BlockIndexEntry> index = new ArrayList<BlockIndexEntry>(indexEntries - 1);
+
+ while (count < (indexEntry.getNumEntries() - interval + 1)) {
+
+ Key myPrevKey = rk.getKey();
+ int pos = cacheBlock.getPosition();
+ rk.readFields(cacheBlock);
+ val.readFields(cacheBlock);
+
+ if (count > 0 && count % interval == 0) {
+ index.add(new BlockIndexEntry(pos, indexEntry.getNumEntries() - count, myPrevKey));
+ }
+
+ count++;
+ }
+
+ this.blockIndex = index.toArray(new BlockIndexEntry[index.size()]);
+
+ cacheBlock.seek(0);
+ }
+
+ BlockIndexEntry[] getIndexEntries() {
+ return blockIndex;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index 7277c65,0000000..7d15851
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@@ -1,971 -1,0 +1,965 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.accumulo.core.file.rfile.bcfile;
+
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile;
+import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile.BlockRead;
+import org.apache.accumulo.core.file.rfile.bcfile.CompareUtils.Scalar;
+import org.apache.accumulo.core.file.rfile.bcfile.CompareUtils.ScalarComparator;
+import org.apache.accumulo.core.file.rfile.bcfile.CompareUtils.ScalarLong;
+import org.apache.accumulo.core.file.rfile.bcfile.Compression.Algorithm;
+import org.apache.accumulo.core.file.rfile.bcfile.Utils.Version;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.compress.Compressor;
+import org.apache.hadoop.io.compress.Decompressor;
+
+/**
+ * Block Compressed file, the underlying physical storage layer for TFile. BCFile provides the basic block level compression for the data block and meta blocks.
+ * It is separated from TFile as it may be used for other block-compressed file implementation.
+ */
+public final class BCFile {
+ // the current version of BCFile impl, increment them (major or minor) made
+ // enough changes
+ static final Version API_VERSION = new Version((short) 1, (short) 0);
+ static final Log LOG = LogFactory.getLog(BCFile.class);
+
+ /**
+ * Prevent the instantiation of BCFile objects.
+ */
+ private BCFile() {
+ // nothing
+ }
+
+ /**
+ * BCFile writer, the entry point for creating a new BCFile.
+ */
+ static public class Writer implements Closeable {
+ private final FSDataOutputStream out;
+ private final Configuration conf;
+ // the single meta block containing index of compressed data blocks
+ final DataIndex dataIndex;
+ // index for meta blocks
+ final MetaIndex metaIndex;
+ boolean blkInProgress = false;
+ private boolean metaBlkSeen = false;
+ private boolean closed = false;
+ long errorCount = 0;
+ // reusable buffers.
+ private BytesWritable fsOutputBuffer;
+
+ /**
+ * Call-back interface to register a block after a block is closed.
+ */
+ private static interface BlockRegister {
+ /**
+ * Register a block that is fully closed.
+ *
+ * @param raw
+ * The size of block in terms of uncompressed bytes.
+ * @param offsetStart
+ * The start offset of the block.
+ * @param offsetEnd
+ * One byte after the end of the block. Compressed block size is offsetEnd - offsetStart.
+ */
+ public void register(long raw, long offsetStart, long offsetEnd);
+ }
+
+ /**
+ * Intermediate class that maintain the state of a Writable Compression Block.
+ */
+ private static final class WBlockState {
+ private final Algorithm compressAlgo;
+ private Compressor compressor; // !null only if using native
+ // Hadoop compression
+ private final FSDataOutputStream fsOut;
+ private final long posStart;
+ private final SimpleBufferedOutputStream fsBufferedOutput;
+ private OutputStream out;
+
+ /**
+ * @param compressionAlgo
+ * The compression algorithm to be used to for compression.
- * @throws IOException
+ */
+ public WBlockState(Algorithm compressionAlgo, FSDataOutputStream fsOut, BytesWritable fsOutputBuffer, Configuration conf) throws IOException {
+ this.compressAlgo = compressionAlgo;
+ this.fsOut = fsOut;
+ this.posStart = fsOut.getPos();
+
+ fsOutputBuffer.setCapacity(TFile.getFSOutputBufferSize(conf));
+
+ this.fsBufferedOutput = new SimpleBufferedOutputStream(this.fsOut, fsOutputBuffer.getBytes());
+ this.compressor = compressAlgo.getCompressor();
+
+ try {
+ this.out = compressionAlgo.createCompressionStream(fsBufferedOutput, compressor, 0);
+ } catch (IOException e) {
+ compressAlgo.returnCompressor(compressor);
+ throw e;
+ }
+ }
+
+ /**
+ * Get the output stream for BlockAppender's consumption.
+ *
+ * @return the output stream suitable for writing block data.
+ */
+ OutputStream getOutputStream() {
+ return out;
+ }
+
+ /**
+ * Get the current position in file.
+ *
+ * @return The current byte offset in underlying file.
+ * @throws IOException
+ */
+ long getCurrentPos() throws IOException {
+ return fsOut.getPos() + fsBufferedOutput.size();
+ }
+
+ long getStartPos() {
+ return posStart;
+ }
+
+ /**
+ * Current size of compressed data.
+ *
+ * @throws IOException
+ */
+ long getCompressedSize() throws IOException {
+ long ret = getCurrentPos() - posStart;
+ return ret;
+ }
+
+ /**
+ * Finishing up the current block.
+ */
+ public void finish() throws IOException {
+ try {
+ if (out != null) {
+ out.flush();
+ out = null;
+ }
+ } finally {
+ compressAlgo.returnCompressor(compressor);
+ compressor = null;
+ }
+ }
+ }
+
+ /**
+ * Access point to stuff data into a block.
+ *
+ */
+ public class BlockAppender extends DataOutputStream {
+ private final BlockRegister blockRegister;
+ private final WBlockState wBlkState;
+ private boolean closed = false;
+
+ /**
+ * Constructor
+ *
+ * @param register
+ * the block register, which is called when the block is closed.
+ * @param wbs
+ * The writable compression block state.
+ */
+ BlockAppender(BlockRegister register, WBlockState wbs) {
+ super(wbs.getOutputStream());
+ this.blockRegister = register;
+ this.wBlkState = wbs;
+ }
+
+ /**
+ * Get the raw size of the block.
+ *
+ * @return the number of uncompressed bytes written through the BlockAppender so far.
- * @throws IOException
+ */
+ public long getRawSize() throws IOException {
+ /**
+ * Expecting the size() of a block not exceeding 4GB. Assuming the size() will wrap to negative integer if it exceeds 2GB.
+ */
+ return size() & 0x00000000ffffffffL;
+ }
+
+ /**
+ * Get the compressed size of the block in progress.
+ *
+ * @return the number of compressed bytes written to the underlying FS file. The size may be smaller than actual need to compress the all data written due
+ * to internal buffering inside the compressor.
- * @throws IOException
+ */
+ public long getCompressedSize() throws IOException {
+ return wBlkState.getCompressedSize();
+ }
+
+ public long getStartPos() {
+ return wBlkState.getStartPos();
+ }
+
+ @Override
+ public void flush() {
+ // The down stream is a special kind of stream that finishes a
+ // compression block upon flush. So we disable flush() here.
+ }
+
+ /**
+ * Signaling the end of write to the block. The block register will be called for registering the finished block.
+ */
+ @Override
+ public void close() throws IOException {
+ if (closed == true) {
+ return;
+ }
+ try {
+ ++errorCount;
+ wBlkState.finish();
+ blockRegister.register(getRawSize(), wBlkState.getStartPos(), wBlkState.getCurrentPos());
+ --errorCount;
+ } finally {
+ closed = true;
+ blkInProgress = false;
+ }
+ }
+ }
+
+ /**
+ * Constructor
+ *
+ * @param fout
+ * FS output stream.
+ * @param compressionName
+ * Name of the compression algorithm, which will be used for all data blocks.
- * @throws IOException
+ * @see Compression#getSupportedAlgorithms
+ */
+ public Writer(FSDataOutputStream fout, String compressionName, Configuration conf, boolean trackDataBlocks) throws IOException {
+ if (fout.getPos() != 0) {
+ throw new IOException("Output file not at zero offset.");
+ }
+
+ this.out = fout;
+ this.conf = conf;
+ dataIndex = new DataIndex(compressionName, trackDataBlocks);
+ metaIndex = new MetaIndex();
+ fsOutputBuffer = new BytesWritable();
+ Magic.write(fout);
+ }
+
+ /**
+ * Close the BCFile Writer. Attempting to use the Writer after calling <code>close</code> is not allowed and may lead to undetermined results.
+ */
++ @Override
+ public void close() throws IOException {
+ if (closed == true) {
+ return;
+ }
+
+ try {
+ if (errorCount == 0) {
+ if (blkInProgress == true) {
+ throw new IllegalStateException("Close() called with active block appender.");
+ }
+
+ // add metaBCFileIndex to metaIndex as the last meta block
+ BlockAppender appender = prepareMetaBlock(DataIndex.BLOCK_NAME, getDefaultCompressionAlgorithm());
+ try {
+ dataIndex.write(appender);
+ } finally {
+ appender.close();
+ }
+
+ long offsetIndexMeta = out.getPos();
+ metaIndex.write(out);
+
+ // Meta Index and the trailing section are written out directly.
+ out.writeLong(offsetIndexMeta);
+
+ API_VERSION.write(out);
+ Magic.write(out);
+ out.flush();
+ }
+ } finally {
+ closed = true;
+ }
+ }
+
+ private Algorithm getDefaultCompressionAlgorithm() {
+ return dataIndex.getDefaultCompressionAlgorithm();
+ }
+
+ private BlockAppender prepareMetaBlock(String name, Algorithm compressAlgo) throws IOException, MetaBlockAlreadyExists {
+ if (blkInProgress == true) {
+ throw new IllegalStateException("Cannot create Meta Block until previous block is closed.");
+ }
+
+ if (metaIndex.getMetaByName(name) != null) {
+ throw new MetaBlockAlreadyExists("name=" + name);
+ }
+
+ MetaBlockRegister mbr = new MetaBlockRegister(name, compressAlgo);
+ WBlockState wbs = new WBlockState(compressAlgo, out, fsOutputBuffer, conf);
+ BlockAppender ba = new BlockAppender(mbr, wbs);
+ blkInProgress = true;
+ metaBlkSeen = true;
+ return ba;
+ }
+
+ /**
+ * Create a Meta Block and obtain an output stream for adding data into the block. There can only be one BlockAppender stream active at any time. Regular
+ * Blocks may not be created after the first Meta Blocks. The caller must call BlockAppender.close() to conclude the block creation.
+ *
+ * @param name
+ * The name of the Meta Block. The name must not conflict with existing Meta Blocks.
+ * @param compressionName
+ * The name of the compression algorithm to be used.
+ * @return The BlockAppender stream
- * @throws IOException
+ * @throws MetaBlockAlreadyExists
+ * If the meta block with the name already exists.
+ */
+ public BlockAppender prepareMetaBlock(String name, String compressionName) throws IOException, MetaBlockAlreadyExists {
+ return prepareMetaBlock(name, Compression.getCompressionAlgorithmByName(compressionName));
+ }
+
+ /**
+ * Create a Meta Block and obtain an output stream for adding data into the block. The Meta Block will be compressed with the same compression algorithm as
+ * data blocks. There can only be one BlockAppender stream active at any time. Regular Blocks may not be created after the first Meta Blocks. The caller
+ * must call BlockAppender.close() to conclude the block creation.
+ *
+ * @param name
+ * The name of the Meta Block. The name must not conflict with existing Meta Blocks.
+ * @return The BlockAppender stream
+ * @throws MetaBlockAlreadyExists
+ * If the meta block with the name already exists.
- * @throws IOException
+ */
+ public BlockAppender prepareMetaBlock(String name) throws IOException, MetaBlockAlreadyExists {
+ return prepareMetaBlock(name, getDefaultCompressionAlgorithm());
+ }
+
+ /**
+ * Create a Data Block and obtain an output stream for adding data into the block. There can only be one BlockAppender stream active at any time. Data
+ * Blocks may not be created after the first Meta Blocks. The caller must call BlockAppender.close() to conclude the block creation.
+ *
+ * @return The BlockAppender stream
- * @throws IOException
+ */
+ public BlockAppender prepareDataBlock() throws IOException {
+ if (blkInProgress == true) {
+ throw new IllegalStateException("Cannot create Data Block until previous block is closed.");
+ }
+
+ if (metaBlkSeen == true) {
+ throw new IllegalStateException("Cannot create Data Block after Meta Blocks.");
+ }
+
+ DataBlockRegister dbr = new DataBlockRegister();
+
+ WBlockState wbs = new WBlockState(getDefaultCompressionAlgorithm(), out, fsOutputBuffer, conf);
+ BlockAppender ba = new BlockAppender(dbr, wbs);
+ blkInProgress = true;
+ return ba;
+ }
+
+ /**
+ * Callback to make sure a meta block is added to the internal list when its stream is closed.
+ */
+ private class MetaBlockRegister implements BlockRegister {
+ private final String name;
+ private final Algorithm compressAlgo;
+
+ MetaBlockRegister(String name, Algorithm compressAlgo) {
+ this.name = name;
+ this.compressAlgo = compressAlgo;
+ }
+
++ @Override
+ public void register(long raw, long begin, long end) {
+ metaIndex.addEntry(new MetaIndexEntry(name, compressAlgo, new BlockRegion(begin, end - begin, raw)));
+ }
+ }
+
+ /**
+ * Callback to make sure a data block is added to the internal list when it's being closed.
+ *
+ */
+ private class DataBlockRegister implements BlockRegister {
+ DataBlockRegister() {
+ // do nothing
+ }
+
++ @Override
+ public void register(long raw, long begin, long end) {
+ dataIndex.addBlockRegion(new BlockRegion(begin, end - begin, raw));
+ }
+ }
+ }
+
+ /**
+ * BCFile Reader, interface to read the file's data and meta blocks.
+ */
+ static public class Reader implements Closeable {
+ private static final String META_NAME = "BCFile.metaindex";
+ private final FSDataInputStream in;
+ private final Configuration conf;
+ final DataIndex dataIndex;
+ // Index for meta blocks
+ final MetaIndex metaIndex;
+ final Version version;
+
+ /**
+ * Intermediate class that maintain the state of a Readable Compression Block.
+ */
+ static private final class RBlockState {
+ private final Algorithm compressAlgo;
+ private Decompressor decompressor;
+ private final BlockRegion region;
+ private final InputStream in;
+
+ public RBlockState(Algorithm compressionAlgo, FSDataInputStream fsin, BlockRegion region, Configuration conf) throws IOException {
+ this.compressAlgo = compressionAlgo;
+ this.region = region;
+ this.decompressor = compressionAlgo.getDecompressor();
+
+ try {
+ this.in = compressAlgo.createDecompressionStream(new BoundedRangeFileInputStream(fsin, this.region.getOffset(), this.region.getCompressedSize()),
+ decompressor, TFile.getFSInputBufferSize(conf));
+ } catch (IOException e) {
+ compressAlgo.returnDecompressor(decompressor);
+ throw e;
+ }
+ }
+
+ /**
+ * Get the output stream for BlockAppender's consumption.
+ *
+ * @return the output stream suitable for writing block data.
+ */
+ public InputStream getInputStream() {
+ return in;
+ }
+
+ public String getCompressionName() {
+ return compressAlgo.getName();
+ }
+
+ public BlockRegion getBlockRegion() {
+ return region;
+ }
+
+ public void finish() throws IOException {
+ try {
+ in.close();
+ } finally {
+ compressAlgo.returnDecompressor(decompressor);
+ decompressor = null;
+ }
+ }
+ }
+
+ /**
+ * Access point to read a block.
+ */
+ public static class BlockReader extends DataInputStream {
+ private final RBlockState rBlkState;
+ private boolean closed = false;
+
+ BlockReader(RBlockState rbs) {
+ super(rbs.getInputStream());
+ rBlkState = rbs;
+ }
+
+ /**
+ * Finishing reading the block. Release all resources.
+ */
+ @Override
+ public void close() throws IOException {
+ if (closed == true) {
+ return;
+ }
+ try {
+ // Do not set rBlkState to null. People may access stats after calling
+ // close().
+ rBlkState.finish();
+ } finally {
+ closed = true;
+ }
+ }
+
+ /**
+ * Get the name of the compression algorithm used to compress the block.
+ *
+ * @return name of the compression algorithm.
+ */
+ public String getCompressionName() {
+ return rBlkState.getCompressionName();
+ }
+
+ /**
+ * Get the uncompressed size of the block.
+ *
+ * @return uncompressed size of the block.
+ */
+ public long getRawSize() {
+ return rBlkState.getBlockRegion().getRawSize();
+ }
+
+ /**
+ * Get the compressed size of the block.
+ *
+ * @return compressed size of the block.
+ */
+ public long getCompressedSize() {
+ return rBlkState.getBlockRegion().getCompressedSize();
+ }
+
+ /**
+ * Get the starting position of the block in the file.
+ *
+ * @return the starting position of the block in the file.
+ */
+ public long getStartPos() {
+ return rBlkState.getBlockRegion().getOffset();
+ }
+ }
+
+ /**
+ * Constructor
+ *
+ * @param fin
+ * FS input stream.
+ * @param fileLength
+ * Length of the corresponding file
- * @throws IOException
+ */
+ public Reader(FSDataInputStream fin, long fileLength, Configuration conf) throws IOException {
+ this.in = fin;
+ this.conf = conf;
+
+ // move the cursor to the beginning of the tail, containing: offset to the
+ // meta block index, version and magic
+ fin.seek(fileLength - Magic.size() - Version.size() - Long.SIZE / Byte.SIZE);
+ long offsetIndexMeta = fin.readLong();
+ version = new Version(fin);
+ Magic.readAndVerify(fin);
+
+ if (!version.compatibleWith(BCFile.API_VERSION)) {
+ throw new RuntimeException("Incompatible BCFile fileBCFileVersion.");
+ }
+
+ // read meta index
+ fin.seek(offsetIndexMeta);
+ metaIndex = new MetaIndex(fin);
+
+ // read data:BCFile.index, the data block index
+ BlockReader blockR = getMetaBlock(DataIndex.BLOCK_NAME);
+ try {
+ dataIndex = new DataIndex(blockR);
+ } finally {
+ blockR.close();
+ }
+ }
+
+ public Reader(CachableBlockFile.Reader cache, FSDataInputStream fin, long fileLength, Configuration conf) throws IOException {
+ this.in = fin;
+ this.conf = conf;
+
+ BlockRead cachedMetaIndex = cache.getCachedMetaBlock(META_NAME);
+ BlockRead cachedDataIndex = cache.getCachedMetaBlock(DataIndex.BLOCK_NAME);
+
+ if (cachedMetaIndex == null || cachedDataIndex == null) {
+ // move the cursor to the beginning of the tail, containing: offset to the
+ // meta block index, version and magic
+ fin.seek(fileLength - Magic.size() - Version.size() - Long.SIZE / Byte.SIZE);
+ long offsetIndexMeta = fin.readLong();
+ version = new Version(fin);
+ Magic.readAndVerify(fin);
+
+ if (!version.compatibleWith(BCFile.API_VERSION)) {
+ throw new RuntimeException("Incompatible BCFile fileBCFileVersion.");
+ }
+
+ // read meta index
+ fin.seek(offsetIndexMeta);
+ metaIndex = new MetaIndex(fin);
+ if (cachedMetaIndex == null) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ metaIndex.write(dos);
+ dos.close();
+ cache.cacheMetaBlock(META_NAME, baos.toByteArray());
+ }
+
+ // read data:BCFile.index, the data block index
+ if (cachedDataIndex == null) {
+ BlockReader blockR = getMetaBlock(DataIndex.BLOCK_NAME);
+ cachedDataIndex = cache.cacheMetaBlock(DataIndex.BLOCK_NAME, blockR);
+ }
+
+ dataIndex = new DataIndex(cachedDataIndex);
+ cachedDataIndex.close();
+
+ } else {
+ // Logger.getLogger(Reader.class).debug("Read bcfile !METADATA from cache");
+ version = null;
+ metaIndex = new MetaIndex(cachedMetaIndex);
+ dataIndex = new DataIndex(cachedDataIndex);
+ }
+ }
+
+ /**
+ * Get the name of the default compression algorithm.
+ *
+ * @return the name of the default compression algorithm.
+ */
+ public String getDefaultCompressionName() {
+ return dataIndex.getDefaultCompressionAlgorithm().getName();
+ }
+
+ /**
+ * Get version of BCFile file being read.
+ *
+ * @return version of BCFile file being read.
+ */
+ public Version getBCFileVersion() {
+ return version;
+ }
+
+ /**
+ * Get version of BCFile API.
+ *
+ * @return version of BCFile API.
+ */
+ public Version getAPIVersion() {
+ return API_VERSION;
+ }
+
+ /**
+ * Finishing reading the BCFile. Release all resources.
+ */
++ @Override
+ public void close() {
+ // nothing to be done now
+ }
+
+ /**
+ * Get the number of data blocks.
+ *
+ * @return the number of data blocks.
+ */
+ public int getBlockCount() {
+ return dataIndex.getBlockRegionList().size();
+ }
+
+ /**
+ * Stream access to a Meta Block.
+ *
+ * @param name
+ * meta block name
+ * @return BlockReader input stream for reading the meta block.
- * @throws IOException
+ * @throws MetaBlockDoesNotExist
+ * The Meta Block with the given name does not exist.
+ */
+ public BlockReader getMetaBlock(String name) throws IOException, MetaBlockDoesNotExist {
+ MetaIndexEntry imeBCIndex = metaIndex.getMetaByName(name);
+ if (imeBCIndex == null) {
+ throw new MetaBlockDoesNotExist("name=" + name);
+ }
+
+ BlockRegion region = imeBCIndex.getRegion();
+ return createReader(imeBCIndex.getCompressionAlgorithm(), region);
+ }
+
+ /**
+ * Stream access to a Data Block.
+ *
+ * @param blockIndex
+ * 0-based data block index.
+ * @return BlockReader input stream for reading the data block.
- * @throws IOException
+ */
+ public BlockReader getDataBlock(int blockIndex) throws IOException {
+ if (blockIndex < 0 || blockIndex >= getBlockCount()) {
+ throw new IndexOutOfBoundsException(String.format("blockIndex=%d, numBlocks=%d", blockIndex, getBlockCount()));
+ }
+
+ BlockRegion region = dataIndex.getBlockRegionList().get(blockIndex);
+ return createReader(dataIndex.getDefaultCompressionAlgorithm(), region);
+ }
+
+ public BlockReader getDataBlock(long offset, long compressedSize, long rawSize) throws IOException {
+ BlockRegion region = new BlockRegion(offset, compressedSize, rawSize);
+ return createReader(dataIndex.getDefaultCompressionAlgorithm(), region);
+ }
+
+ private BlockReader createReader(Algorithm compressAlgo, BlockRegion region) throws IOException {
+ RBlockState rbs = new RBlockState(compressAlgo, in, region, conf);
+ return new BlockReader(rbs);
+ }
+
+ /**
+ * Find the smallest Block index whose starting offset is greater than or equal to the specified offset.
+ *
+ * @param offset
+ * User-specific offset.
+ * @return the index to the data Block if such block exists; or -1 otherwise.
+ */
+ public int getBlockIndexNear(long offset) {
+ ArrayList<BlockRegion> list = dataIndex.getBlockRegionList();
+ int idx = Utils.lowerBound(list, new ScalarLong(offset), new ScalarComparator());
+
+ if (idx == list.size()) {
+ return -1;
+ }
+
+ return idx;
+ }
+ }
+
+ /**
+ * Index for all Meta blocks.
+ */
+ static class MetaIndex {
+ // use a tree map, for getting a meta block entry by name
+ final Map<String,MetaIndexEntry> index;
+
+ // for write
+ public MetaIndex() {
+ index = new TreeMap<String,MetaIndexEntry>();
+ }
+
+ // for read, construct the map from the file
+ public MetaIndex(DataInput in) throws IOException {
+ int count = Utils.readVInt(in);
+ index = new TreeMap<String,MetaIndexEntry>();
+
+ for (int nx = 0; nx < count; nx++) {
+ MetaIndexEntry indexEntry = new MetaIndexEntry(in);
+ index.put(indexEntry.getMetaName(), indexEntry);
+ }
+ }
+
+ public void addEntry(MetaIndexEntry indexEntry) {
+ index.put(indexEntry.getMetaName(), indexEntry);
+ }
+
+ public MetaIndexEntry getMetaByName(String name) {
+ return index.get(name);
+ }
+
+ public void write(DataOutput out) throws IOException {
+ Utils.writeVInt(out, index.size());
+
+ for (MetaIndexEntry indexEntry : index.values()) {
+ indexEntry.write(out);
+ }
+ }
+ }
+
+ /**
+ * An entry describes a meta block in the MetaIndex.
+ */
+ static final class MetaIndexEntry {
+ private final String metaName;
+ private final Algorithm compressionAlgorithm;
+ private final static String defaultPrefix = "data:";
+
+ private final BlockRegion region;
+
+ public MetaIndexEntry(DataInput in) throws IOException {
+ String fullMetaName = Utils.readString(in);
+ if (fullMetaName.startsWith(defaultPrefix)) {
+ metaName = fullMetaName.substring(defaultPrefix.length(), fullMetaName.length());
+ } else {
+ throw new IOException("Corrupted Meta region Index");
+ }
+
+ compressionAlgorithm = Compression.getCompressionAlgorithmByName(Utils.readString(in));
+ region = new BlockRegion(in);
+ }
+
+ public MetaIndexEntry(String metaName, Algorithm compressionAlgorithm, BlockRegion region) {
+ this.metaName = metaName;
+ this.compressionAlgorithm = compressionAlgorithm;
+ this.region = region;
+ }
+
+ public String getMetaName() {
+ return metaName;
+ }
+
+ public Algorithm getCompressionAlgorithm() {
+ return compressionAlgorithm;
+ }
+
+ public BlockRegion getRegion() {
+ return region;
+ }
+
+ public void write(DataOutput out) throws IOException {
+ Utils.writeString(out, defaultPrefix + metaName);
+ Utils.writeString(out, compressionAlgorithm.getName());
+
+ region.write(out);
+ }
+ }
+
+ /**
+ * Index of all compressed data blocks.
+ */
+ static class DataIndex {
+ final static String BLOCK_NAME = "BCFile.index";
+
+ private final Algorithm defaultCompressionAlgorithm;
+
+ // for data blocks, each entry specifies a block's offset, compressed size
+ // and raw size
+ private final ArrayList<BlockRegion> listRegions;
+
+ private boolean trackBlocks;
+
+ // for read, deserialized from a file
+ public DataIndex(DataInput in) throws IOException {
+ defaultCompressionAlgorithm = Compression.getCompressionAlgorithmByName(Utils.readString(in));
+
+ int n = Utils.readVInt(in);
+ listRegions = new ArrayList<BlockRegion>(n);
+
+ for (int i = 0; i < n; i++) {
+ BlockRegion region = new BlockRegion(in);
+ listRegions.add(region);
+ }
+ }
+
+ // for write
+ public DataIndex(String defaultCompressionAlgorithmName, boolean trackBlocks) {
+ this.trackBlocks = trackBlocks;
+ this.defaultCompressionAlgorithm = Compression.getCompressionAlgorithmByName(defaultCompressionAlgorithmName);
+ listRegions = new ArrayList<BlockRegion>();
+ }
+
+ public Algorithm getDefaultCompressionAlgorithm() {
+ return defaultCompressionAlgorithm;
+ }
+
+ public ArrayList<BlockRegion> getBlockRegionList() {
+ return listRegions;
+ }
+
+ public void addBlockRegion(BlockRegion region) {
+ if (trackBlocks)
+ listRegions.add(region);
+ }
+
+ public void write(DataOutput out) throws IOException {
+ Utils.writeString(out, defaultCompressionAlgorithm.getName());
+
+ Utils.writeVInt(out, listRegions.size());
+
+ for (BlockRegion region : listRegions) {
+ region.write(out);
+ }
+ }
+ }
+
+ /**
+ * Magic number uniquely identifying a BCFile in the header/footer.
+ */
+ static final class Magic {
+ private final static byte[] AB_MAGIC_BCFILE = {
+ // ... total of 16 bytes
+ (byte) 0xd1, (byte) 0x11, (byte) 0xd3, (byte) 0x68, (byte) 0x91, (byte) 0xb5, (byte) 0xd7, (byte) 0xb6, (byte) 0x39, (byte) 0xdf, (byte) 0x41,
+ (byte) 0x40, (byte) 0x92, (byte) 0xba, (byte) 0xe1, (byte) 0x50};
+
+ public static void readAndVerify(DataInput in) throws IOException {
+ byte[] abMagic = new byte[size()];
+ in.readFully(abMagic);
+
+ // check against AB_MAGIC_BCFILE, if not matching, throw an
+ // Exception
+ if (!Arrays.equals(abMagic, AB_MAGIC_BCFILE)) {
+ throw new IOException("Not a valid BCFile.");
+ }
+ }
+
+ public static void write(DataOutput out) throws IOException {
+ out.write(AB_MAGIC_BCFILE);
+ }
+
+ public static int size() {
+ return AB_MAGIC_BCFILE.length;
+ }
+ }
+
+ /**
+ * Block region.
+ */
+ static final class BlockRegion implements Scalar {
+ private final long offset;
+ private final long compressedSize;
+ private final long rawSize;
+
+ public BlockRegion(DataInput in) throws IOException {
+ offset = Utils.readVLong(in);
+ compressedSize = Utils.readVLong(in);
+ rawSize = Utils.readVLong(in);
+ }
+
+ public BlockRegion(long offset, long compressedSize, long rawSize) {
+ this.offset = offset;
+ this.compressedSize = compressedSize;
+ this.rawSize = rawSize;
+ }
+
+ public void write(DataOutput out) throws IOException {
+ Utils.writeVLong(out, offset);
+ Utils.writeVLong(out, compressedSize);
+ Utils.writeVLong(out, rawSize);
+ }
+
+ public long getOffset() {
+ return offset;
+ }
+
+ public long getCompressedSize() {
+ return compressedSize;
+ }
+
+ public long getRawSize() {
+ return rawSize;
+ }
+
+ @Override
+ public long magnitude() {
+ return offset;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/ByteArray.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/ByteArray.java
index 2b57638,0000000..d7734a2
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/ByteArray.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/ByteArray.java
@@@ -1,91 -1,0 +1,89 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.accumulo.core.file.rfile.bcfile;
+
+import org.apache.hadoop.io.BytesWritable;
+
+/**
+ * Adaptor class to wrap byte-array backed objects (including java byte array) as RawComparable objects.
+ */
+public final class ByteArray implements RawComparable {
+ private final byte[] buffer;
+ private final int offset;
+ private final int len;
+
+ /**
+ * Constructing a ByteArray from a {@link BytesWritable}.
- *
- * @param other
+ */
+ public ByteArray(BytesWritable other) {
+ this(other.getBytes(), 0, other.getLength());
+ }
+
+ /**
+ * Wrap a whole byte array as a RawComparable.
+ *
+ * @param buffer
+ * the byte array buffer.
+ */
+ public ByteArray(byte[] buffer) {
+ this(buffer, 0, buffer.length);
+ }
+
+ /**
+ * Wrap a partial byte array as a RawComparable.
+ *
+ * @param buffer
+ * the byte array buffer.
+ * @param offset
+ * the starting offset
+ * @param len
+ * the length of the consecutive bytes to be wrapped.
+ */
+ public ByteArray(byte[] buffer, int offset, int len) {
+ if ((offset | len | (buffer.length - offset - len)) < 0) {
+ throw new IndexOutOfBoundsException();
+ }
+ this.buffer = buffer;
+ this.offset = offset;
+ this.len = len;
+ }
+
+ /**
+ * @return the underlying buffer.
+ */
+ @Override
+ public byte[] buffer() {
+ return buffer;
+ }
+
+ /**
+ * @return the offset in the buffer.
+ */
+ @Override
+ public int offset() {
+ return offset;
+ }
+
+ /**
+ * @return the size of the byte array.
+ */
+ @Override
+ public int size() {
+ return len;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Chunk.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Chunk.java
index a075d87,0000000..345d406
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Chunk.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Chunk.java
@@@ -1,418 -1,0 +1,416 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.accumulo.core.file.rfile.bcfile;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Several related classes to support chunk-encoded sub-streams on top of a regular stream.
+ */
+final class Chunk {
+
+ /**
+ * Prevent the instantiation of class.
+ */
+ private Chunk() {
+ // nothing
+ }
+
+ /**
+ * Decoding a chain of chunks encoded through ChunkEncoder or SingleChunkEncoder.
+ */
+ static public class ChunkDecoder extends InputStream {
+ private DataInputStream in = null;
+ private boolean lastChunk;
+ private int remain = 0;
+ private boolean closed;
+
+ public ChunkDecoder() {
+ lastChunk = true;
+ closed = true;
+ }
+
+ public void reset(DataInputStream downStream) {
+ // no need to wind forward the old input.
+ in = downStream;
+ lastChunk = false;
+ remain = 0;
+ closed = false;
+ }
+
+ /**
+ * Constructor
+ *
+ * @param in
+ * The source input stream which contains chunk-encoded data stream.
+ */
+ public ChunkDecoder(DataInputStream in) {
+ this.in = in;
+ lastChunk = false;
+ closed = false;
+ }
+
+ /**
+ * Have we reached the last chunk.
+ *
+ * @return true if we have reached the last chunk.
- * @throws java.io.IOException
+ */
+ public boolean isLastChunk() throws IOException {
+ checkEOF();
+ return lastChunk;
+ }
+
+ /**
+ * How many bytes remain in the current chunk?
+ *
+ * @return remaining bytes left in the current chunk.
- * @throws java.io.IOException
+ */
+ public int getRemain() throws IOException {
+ checkEOF();
+ return remain;
+ }
+
+ /**
+ * Reading the length of next chunk.
+ *
+ * @throws java.io.IOException
+ * when no more data is available.
+ */
+ private void readLength() throws IOException {
+ remain = Utils.readVInt(in);
+ if (remain >= 0) {
+ lastChunk = true;
+ } else {
+ remain = -remain;
+ }
+ }
+
+ /**
+ * Check whether we reach the end of the stream.
+ *
+ * @return false if the chunk encoded stream has more data to read (in which case available() will be greater than 0); true otherwise.
+ * @throws java.io.IOException
+ * on I/O errors.
+ */
+ private boolean checkEOF() throws IOException {
+ if (isClosed())
+ return true;
+ while (true) {
+ if (remain > 0)
+ return false;
+ if (lastChunk)
+ return true;
+ readLength();
+ }
+ }
+
+ @Override
+ /*
+ * This method never blocks the caller. Returning 0 does not mean we reach the end of the stream.
+ */
+ public int available() {
+ return remain;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (checkEOF())
+ return -1;
+ int ret = in.read();
+ if (ret < 0)
+ throw new IOException("Corrupted chunk encoding stream");
+ --remain;
+ return ret;
+ }
+
+ @Override
+ public int read(byte[] b) throws IOException {
+ return read(b, 0, b.length);
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ if ((off | len | (off + len) | (b.length - (off + len))) < 0) {
+ throw new IndexOutOfBoundsException();
+ }
+
+ if (!checkEOF()) {
+ int n = Math.min(remain, len);
+ int ret = in.read(b, off, n);
+ if (ret < 0)
+ throw new IOException("Corrupted chunk encoding stream");
+ remain -= ret;
+ return ret;
+ }
+ return -1;
+ }
+
+ @Override
+ public long skip(long n) throws IOException {
+ if (!checkEOF()) {
+ long ret = in.skip(Math.min(remain, n));
+ remain -= ret;
+ return ret;
+ }
+ return 0;
+ }
+
+ @Override
+ public boolean markSupported() {
+ return false;
+ }
+
+ public boolean isClosed() {
+ return closed;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed == false) {
+ try {
+ while (!checkEOF()) {
+ skip(Integer.MAX_VALUE);
+ }
+ } finally {
+ closed = true;
+ }
+ }
+ }
+ }
+
+ /**
+ * Chunk Encoder. Encoding the output data into a chain of chunks in the following sequences: -len1, byte[len1], -len2, byte[len2], ... len_n, byte[len_n].
+ * Where len1, len2, ..., len_n are the lengths of the data chunks. Non-terminal chunks have their lengths negated. Non-terminal chunks cannot have length 0.
+ * All lengths are in the range of 0 to Integer.MAX_VALUE and are encoded in Utils.VInt format.
+ */
+ static public class ChunkEncoder extends OutputStream {
+ /**
+ * The data output stream it connects to.
+ */
+ private DataOutputStream out;
+
+ /**
+ * The internal buffer that is only used when we do not know the advertised size.
+ */
+ private byte buf[];
+
+ /**
+ * The number of valid bytes in the buffer. This value is always in the range <tt>0</tt> through <tt>buf.length</tt>; elements <tt>buf[0]</tt> through
+ * <tt>buf[count-1]</tt> contain valid byte data.
+ */
+ private int count;
+
+ /**
+ * Constructor.
+ *
+ * @param out
+ * the underlying output stream.
+ * @param buf
+ * user-supplied buffer. The buffer would be used exclusively by the ChunkEncoder during its life cycle.
+ */
+ public ChunkEncoder(DataOutputStream out, byte[] buf) {
+ this.out = out;
+ this.buf = buf;
+ this.count = 0;
+ }
+
+ /**
+ * Write out a chunk.
+ *
+ * @param chunk
+ * The chunk buffer.
+ * @param offset
+ * Offset to chunk buffer for the beginning of chunk.
+ * @param len
+ * @param last
+ * Is this the last call to flushBuffer?
+ */
+ private void writeChunk(byte[] chunk, int offset, int len, boolean last) throws IOException {
+ if (last) { // always write out the length for the last chunk.
+ Utils.writeVInt(out, len);
+ if (len > 0) {
+ out.write(chunk, offset, len);
+ }
+ } else {
+ if (len > 0) {
+ Utils.writeVInt(out, -len);
+ out.write(chunk, offset, len);
+ }
+ }
+ }
+
+ /**
+ * Write out a chunk that is a concatenation of the internal buffer plus user supplied data. This will never be the last block.
+ *
+ * @param data
+ * User supplied data buffer.
+ * @param offset
+ * Offset to user data buffer.
+ * @param len
+ * User data buffer size.
+ */
+ private void writeBufData(byte[] data, int offset, int len) throws IOException {
+ if (count + len > 0) {
+ Utils.writeVInt(out, -(count + len));
+ out.write(buf, 0, count);
+ count = 0;
+ out.write(data, offset, len);
+ }
+ }
+
+ /**
+ * Flush the internal buffer.
+ *
+ * Is this the last call to flushBuffer?
+ *
+ * @throws java.io.IOException
+ */
+ private void flushBuffer() throws IOException {
+ if (count > 0) {
+ writeChunk(buf, 0, count, false);
+ count = 0;
+ }
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ if (count >= buf.length) {
+ flushBuffer();
+ }
+ buf[count++] = (byte) b;
+ }
+
+ @Override
+ public void write(byte b[]) throws IOException {
+ write(b, 0, b.length);
+ }
+
+ @Override
+ public void write(byte b[], int off, int len) throws IOException {
+ if ((len + count) >= buf.length) {
+ /*
+ * If the input data do not fit in buffer, flush the output buffer and then write the data directly. In this way buffered streams will cascade
+ * harmlessly.
+ */
+ writeBufData(b, off, len);
+ return;
+ }
+
+ System.arraycopy(b, off, buf, count, len);
+ count += len;
+ }
+
+ @Override
+ public void flush() throws IOException {
+ flushBuffer();
+ out.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (buf != null) {
+ try {
+ writeChunk(buf, 0, count, true);
+ } finally {
+ buf = null;
+ out = null;
+ }
+ }
+ }
+ }
+
+ /**
+ * Encode the whole stream as a single chunk. Expecting to know the size of the chunk up-front.
+ */
+ static public class SingleChunkEncoder extends OutputStream {
+ /**
+ * The data output stream it connects to.
+ */
+ private final DataOutputStream out;
+
+ /**
+ * The remaining bytes to be written.
+ */
+ private int remain;
+ private boolean closed = false;
+
+ /**
+ * Constructor.
+ *
+ * @param out
+ * the underlying output stream.
+ * @param size
+ * The total # of bytes to be written as a single chunk.
+ * @throws java.io.IOException
+ * if an I/O error occurs.
+ */
+ public SingleChunkEncoder(DataOutputStream out, int size) throws IOException {
+ this.out = out;
+ this.remain = size;
+ Utils.writeVInt(out, size);
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ if (remain > 0) {
+ out.write(b);
+ --remain;
+ } else {
+ throw new IOException("Writing more bytes than advertised size.");
+ }
+ }
+
+ @Override
+ public void write(byte b[]) throws IOException {
+ write(b, 0, b.length);
+ }
+
+ @Override
+ public void write(byte b[], int off, int len) throws IOException {
+ if (remain >= len) {
+ out.write(b, off, len);
+ remain -= len;
+ } else {
+ throw new IOException("Writing more bytes than advertised size.");
+ }
+ }
+
+ @Override
+ public void flush() throws IOException {
+ out.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed == true) {
+ return;
+ }
+
+ try {
+ if (remain > 0) {
+ throw new IOException("Writing less bytes than advertised size.");
+ }
+ } finally {
+ closed = true;
+ }
+ }
+ }
+}
[53/64] [abbrv] Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 6d73125,0000000..e2510d7
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@@ -1,3913 -1,0 +1,3913 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver;
+
+import static org.apache.accumulo.server.problems.ProblemType.TABLET_LOAD;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.net.Socket;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TimerTask;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.RunnableFuture;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.management.ObjectName;
+import javax.management.StandardMBean;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.CompressedIterators;
+import org.apache.accumulo.core.client.impl.CompressedIterators.IterConfig;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.TabletType;
+import org.apache.accumulo.core.client.impl.Translator;
+import org.apache.accumulo.core.client.impl.Translator.TKeyExtentTranslator;
+import org.apache.accumulo.core.client.impl.Translator.TRangeTranslator;
++import org.apache.accumulo.core.client.impl.Translators;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
- import org.apache.accumulo.core.client.impl.Translators;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.constraints.Constraint.Environment;
+import org.apache.accumulo.core.constraints.Violations;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Column;
+import org.apache.accumulo.core.data.ConstraintViolationSummary;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.thrift.InitialMultiScan;
+import org.apache.accumulo.core.data.thrift.InitialScan;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.data.thrift.MapFileInfo;
+import org.apache.accumulo.core.data.thrift.MultiScanResult;
+import org.apache.accumulo.core.data.thrift.ScanResult;
+import org.apache.accumulo.core.data.thrift.TCMResult;
+import org.apache.accumulo.core.data.thrift.TCMStatus;
+import org.apache.accumulo.core.data.thrift.TColumn;
+import org.apache.accumulo.core.data.thrift.TCondition;
+import org.apache.accumulo.core.data.thrift.TConditionalMutation;
+import org.apache.accumulo.core.data.thrift.TConditionalSession;
+import org.apache.accumulo.core.data.thrift.TKey;
+import org.apache.accumulo.core.data.thrift.TKeyExtent;
+import org.apache.accumulo.core.data.thrift.TKeyValue;
+import org.apache.accumulo.core.data.thrift.TMutation;
+import org.apache.accumulo.core.data.thrift.TRange;
+import org.apache.accumulo.core.data.thrift.UpdateErrors;
+import org.apache.accumulo.core.iterators.IterationInterruptedException;
+import org.apache.accumulo.core.master.thrift.Compacting;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.master.thrift.TableInfo;
+import org.apache.accumulo.core.master.thrift.TabletLoadState;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.security.AuthorizationContainer;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.tabletserver.log.LogEntry;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
+import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
+import org.apache.accumulo.core.tabletserver.thrift.NoSuchScanIDException;
+import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
+import org.apache.accumulo.core.tabletserver.thrift.ScanState;
+import org.apache.accumulo.core.tabletserver.thrift.ScanType;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Processor;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.LoggingRunnable;
+import org.apache.accumulo.core.util.MapCounter;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.ServerServices;
+import org.apache.accumulo.core.util.ServerServices.Service;
+import org.apache.accumulo.core.util.SimpleThreadPool;
+import org.apache.accumulo.core.util.Stat;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockWatcher;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.ServerOpts;
+import org.apache.accumulo.server.client.ClientServiceHandler;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.conf.TableConfiguration;
+import org.apache.accumulo.server.data.ServerMutation;
+import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManager.FileType;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.fs.VolumeUtil;
+import org.apache.accumulo.server.master.recovery.RecoveryPath;
+import org.apache.accumulo.server.master.state.Assignment;
+import org.apache.accumulo.server.master.state.DistributedStoreException;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletLocationState;
+import org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException;
+import org.apache.accumulo.server.master.state.TabletStateStore;
+import org.apache.accumulo.server.master.state.ZooTabletStateStore;
+import org.apache.accumulo.server.metrics.AbstractMetricsImpl;
+import org.apache.accumulo.server.problems.ProblemReport;
+import org.apache.accumulo.server.problems.ProblemReports;
+import org.apache.accumulo.server.security.AuditedSecurityOperation;
+import org.apache.accumulo.server.security.SecurityOperation;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.util.FileSystemMonitor;
+import org.apache.accumulo.server.util.Halt;
+import org.apache.accumulo.server.util.MasterMetadataUtil;
+import org.apache.accumulo.server.util.MetadataTableUtil;
+import org.apache.accumulo.server.util.TServerUtils;
+import org.apache.accumulo.server.util.TServerUtils.ServerAddress;
+import org.apache.accumulo.server.util.time.RelativeTime;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
+import org.apache.accumulo.server.zookeeper.TransactionWatcher;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.accumulo.start.classloader.vfs.ContextManager;
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.instrument.Trace;
+import org.apache.accumulo.trace.instrument.thrift.TraceWrap;
+import org.apache.accumulo.trace.thrift.TInfo;
+import org.apache.accumulo.tserver.Compactor.CompactionInfo;
+import org.apache.accumulo.tserver.RowLocks.RowLock;
+import org.apache.accumulo.tserver.Tablet.CommitSession;
+import org.apache.accumulo.tserver.Tablet.KVEntry;
+import org.apache.accumulo.tserver.Tablet.LookupResult;
+import org.apache.accumulo.tserver.Tablet.MinorCompactionReason;
+import org.apache.accumulo.tserver.Tablet.ScanBatch;
+import org.apache.accumulo.tserver.Tablet.Scanner;
+import org.apache.accumulo.tserver.Tablet.SplitInfo;
+import org.apache.accumulo.tserver.Tablet.TConstraintViolationException;
+import org.apache.accumulo.tserver.Tablet.TabletClosedException;
+import org.apache.accumulo.tserver.TabletServerResourceManager.TabletResourceManager;
+import org.apache.accumulo.tserver.TabletStatsKeeper.Operation;
+import org.apache.accumulo.tserver.compaction.MajorCompactionReason;
+import org.apache.accumulo.tserver.data.ServerConditionalMutation;
+import org.apache.accumulo.tserver.log.DfsLogger;
+import org.apache.accumulo.tserver.log.LogSorter;
+import org.apache.accumulo.tserver.log.MutationReceiver;
+import org.apache.accumulo.tserver.log.TabletServerLogger;
+import org.apache.accumulo.tserver.mastermessage.MasterMessage;
+import org.apache.accumulo.tserver.mastermessage.SplitReportMessage;
+import org.apache.accumulo.tserver.mastermessage.TabletStatusMessage;
+import org.apache.accumulo.tserver.metrics.TabletServerMBean;
+import org.apache.accumulo.tserver.metrics.TabletServerMinCMetrics;
+import org.apache.accumulo.tserver.metrics.TabletServerScanMetrics;
+import org.apache.accumulo.tserver.metrics.TabletServerUpdateMetrics;
+import org.apache.commons.collections.map.LRUMap;
+import org.apache.hadoop.fs.FSError;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+import org.apache.thrift.TProcessor;
+import org.apache.thrift.TServiceClient;
+import org.apache.thrift.server.TServer;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+
+import com.google.common.net.HostAndPort;
+
+enum ScanRunState {
+ QUEUED, RUNNING, FINISHED
+}
+
+public class TabletServer extends AbstractMetricsImpl implements org.apache.accumulo.tserver.metrics.TabletServerMBean {
+ private static final Logger log = Logger.getLogger(TabletServer.class);
+
+ private static HashMap<String,Long> prevGcTime = new HashMap<String,Long>();
+ private static long lastMemorySize = 0;
+ private static long gcTimeIncreasedCount;
+
+ private static final long MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS = 1000;
+ private static final long RECENTLY_SPLIT_MILLIES = 60 * 1000;
+
+ private TabletServerLogger logger;
+
+ protected TabletServerMinCMetrics mincMetrics = new TabletServerMinCMetrics();
+
+ private ServerConfiguration serverConfig;
+ private LogSorter logSorter = null;
+
+ public TabletServer(ServerConfiguration conf, VolumeManager fs) {
+ super();
+ this.serverConfig = conf;
+ this.instance = conf.getInstance();
+ this.fs = fs;
+ this.logSorter = new LogSorter(instance, fs, getSystemConfiguration());
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ synchronized (onlineTablets) {
+ long now = System.currentTimeMillis();
+ for (Tablet tablet : onlineTablets.values())
+ try {
+ tablet.updateRates(now);
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ }
+ }
+ }, 5000, 5000);
+ }
+
+ private synchronized static void logGCInfo(AccumuloConfiguration conf) {
+ List<GarbageCollectorMXBean> gcmBeans = ManagementFactory.getGarbageCollectorMXBeans();
+ Runtime rt = Runtime.getRuntime();
+
+ StringBuilder sb = new StringBuilder("gc");
+
+ boolean sawChange = false;
+
+ long maxIncreaseInCollectionTime = 0;
+
+ for (GarbageCollectorMXBean gcBean : gcmBeans) {
+ Long prevTime = prevGcTime.get(gcBean.getName());
+ long pt = 0;
+ if (prevTime != null) {
+ pt = prevTime;
+ }
+
+ long time = gcBean.getCollectionTime();
+
+ if (time - pt != 0) {
+ sawChange = true;
+ }
+
+ long increaseInCollectionTime = time - pt;
+ sb.append(String.format(" %s=%,.2f(+%,.2f) secs", gcBean.getName(), time / 1000.0, increaseInCollectionTime / 1000.0));
+ maxIncreaseInCollectionTime = Math.max(increaseInCollectionTime, maxIncreaseInCollectionTime);
+ prevGcTime.put(gcBean.getName(), time);
+ }
+
+ long mem = rt.freeMemory();
+ if (maxIncreaseInCollectionTime == 0) {
+ gcTimeIncreasedCount = 0;
+ } else {
+ gcTimeIncreasedCount++;
+ if (gcTimeIncreasedCount > 3 && mem < rt.maxMemory() * 0.05) {
+ log.warn("Running low on memory");
+ gcTimeIncreasedCount = 0;
+ }
+ }
+
+ if (mem > lastMemorySize) {
+ sawChange = true;
+ }
+
+ String sign = "+";
+ if (mem - lastMemorySize <= 0) {
+ sign = "";
+ }
+
+ sb.append(String.format(" freemem=%,d(%s%,d) totalmem=%,d", mem, sign, (mem - lastMemorySize), rt.totalMemory()));
+
+ if (sawChange) {
+ log.debug(sb.toString());
+ }
+
+ final long keepAliveTimeout = conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT);
+ if (maxIncreaseInCollectionTime > keepAliveTimeout) {
+ Halt.halt("Garbage collection may be interfering with lock keep-alive. Halting.", -1);
+ }
+
+ lastMemorySize = mem;
+ }
+
+ private TabletStatsKeeper statsKeeper;
+
+ private static class Session {
+ long lastAccessTime;
+ long startTime;
+ String user;
+ String client = TServerUtils.clientAddress.get();
+ public boolean reserved;
+
+ public void cleanup() {}
+ }
+
+ private static class SessionManager {
+
+ SecureRandom random;
+ Map<Long,Session> sessions;
+ long maxIdle;
+
+ SessionManager(AccumuloConfiguration conf) {
+ random = new SecureRandom();
+ sessions = new HashMap<Long,Session>();
+
+ maxIdle = conf.getTimeInMillis(Property.TSERV_SESSION_MAXIDLE);
+
+ Runnable r = new Runnable() {
+ @Override
+ public void run() {
+ sweep(maxIdle);
+ }
+ };
+
+ SimpleTimer.getInstance().schedule(r, 0, Math.max(maxIdle / 2, 1000));
+ }
+
+ synchronized long createSession(Session session, boolean reserve) {
+ long sid = random.nextLong();
+
+ while (sessions.containsKey(sid)) {
+ sid = random.nextLong();
+ }
+
+ sessions.put(sid, session);
+
+ session.reserved = reserve;
+
+ session.startTime = session.lastAccessTime = System.currentTimeMillis();
+
+ return sid;
+ }
+
+ long getMaxIdleTime() {
+ return maxIdle;
+ }
+
+ /**
+ * while a session is reserved, it cannot be canceled or removed
- *
++ *
+ * @param sessionId
+ */
+
+ synchronized Session reserveSession(long sessionId) {
+ Session session = sessions.get(sessionId);
+ if (session != null) {
+ if (session.reserved)
+ throw new IllegalStateException();
+ session.reserved = true;
+ }
+
+ return session;
+
+ }
+
+ synchronized Session reserveSession(long sessionId, boolean wait) {
+ Session session = sessions.get(sessionId);
+ if (session != null) {
+ while (wait && session.reserved) {
+ try {
+ wait(1000);
+ } catch (InterruptedException e) {
+ throw new RuntimeException();
+ }
+ }
+
+ if (session.reserved)
+ throw new IllegalStateException();
+ session.reserved = true;
+ }
+
+ return session;
+
+ }
+
+ synchronized void unreserveSession(Session session) {
+ if (!session.reserved)
+ throw new IllegalStateException();
+ notifyAll();
+ session.reserved = false;
+ session.lastAccessTime = System.currentTimeMillis();
+ }
+
+ synchronized void unreserveSession(long sessionId) {
+ Session session = getSession(sessionId);
+ if (session != null)
+ unreserveSession(session);
+ }
+
+ synchronized Session getSession(long sessionId) {
+ Session session = sessions.get(sessionId);
+ if (session != null)
+ session.lastAccessTime = System.currentTimeMillis();
+ return session;
+ }
+
+ Session removeSession(long sessionId) {
+ return removeSession(sessionId, false);
+ }
+
+ Session removeSession(long sessionId, boolean unreserve) {
+ Session session = null;
+ synchronized (this) {
+ session = sessions.remove(sessionId);
+ if (unreserve && session != null)
+ unreserveSession(session);
+ }
+
+ // do clean up out side of lock..
+ if (session != null)
+ session.cleanup();
+
+ return session;
+ }
+
+ private void sweep(long maxIdle) {
+ ArrayList<Session> sessionsToCleanup = new ArrayList<Session>();
+ synchronized (this) {
+ Iterator<Session> iter = sessions.values().iterator();
+ while (iter.hasNext()) {
+ Session session = iter.next();
+ long idleTime = System.currentTimeMillis() - session.lastAccessTime;
+ if (idleTime > maxIdle && !session.reserved) {
+ iter.remove();
+ sessionsToCleanup.add(session);
+ }
+ }
+ }
+
+ // do clean up outside of lock
+ for (Session session : sessionsToCleanup) {
+ session.cleanup();
+ }
+ }
+
+ synchronized void removeIfNotAccessed(final long sessionId, long delay) {
+ Session session = sessions.get(sessionId);
+ if (session != null) {
+ final long removeTime = session.lastAccessTime;
+ TimerTask r = new TimerTask() {
+ @Override
+ public void run() {
+ Session sessionToCleanup = null;
+ synchronized (SessionManager.this) {
+ Session session2 = sessions.get(sessionId);
+ if (session2 != null && session2.lastAccessTime == removeTime && !session2.reserved) {
+ sessions.remove(sessionId);
+ sessionToCleanup = session2;
+ }
+ }
+
+ // call clean up outside of lock
+ if (sessionToCleanup != null)
+ sessionToCleanup.cleanup();
+ }
+ };
+
+ SimpleTimer.getInstance().schedule(r, delay);
+ }
+ }
+
+ public synchronized Map<String,MapCounter<ScanRunState>> getActiveScansPerTable() {
+ Map<String,MapCounter<ScanRunState>> counts = new HashMap<String,MapCounter<ScanRunState>>();
+ for (Entry<Long,Session> entry : sessions.entrySet()) {
+
+ Session session = entry.getValue();
+ @SuppressWarnings("rawtypes")
+ ScanTask nbt = null;
+ String tableID = null;
+
+ if (session instanceof ScanSession) {
+ ScanSession ss = (ScanSession) session;
+ nbt = ss.nextBatchTask;
+ tableID = ss.extent.getTableId().toString();
+ } else if (session instanceof MultiScanSession) {
+ MultiScanSession mss = (MultiScanSession) session;
+ nbt = mss.lookupTask;
+ tableID = mss.threadPoolExtent.getTableId().toString();
+ }
+
+ if (nbt == null)
+ continue;
+
+ ScanRunState srs = nbt.getScanRunState();
+
+ if (srs == ScanRunState.FINISHED)
+ continue;
+
+ MapCounter<ScanRunState> stateCounts = counts.get(tableID);
+ if (stateCounts == null) {
+ stateCounts = new MapCounter<ScanRunState>();
+ counts.put(tableID, stateCounts);
+ }
+
+ stateCounts.increment(srs, 1);
+ }
+
+ return counts;
+ }
+
+ public synchronized List<ActiveScan> getActiveScans() {
+
+ ArrayList<ActiveScan> activeScans = new ArrayList<ActiveScan>();
+
+ long ct = System.currentTimeMillis();
+
+ for (Entry<Long,Session> entry : sessions.entrySet()) {
+ Session session = entry.getValue();
+ if (session instanceof ScanSession) {
+ ScanSession ss = (ScanSession) session;
+
+ ScanState state = ScanState.RUNNING;
+
+ ScanTask<ScanBatch> nbt = ss.nextBatchTask;
+ if (nbt == null) {
+ state = ScanState.IDLE;
+ } else {
+ switch (nbt.getScanRunState()) {
+ case QUEUED:
+ state = ScanState.QUEUED;
+ break;
+ case FINISHED:
+ state = ScanState.IDLE;
+ break;
+ case RUNNING:
+ default:
+ /* do nothing */
+ break;
+ }
+ }
+
+ activeScans.add(new ActiveScan(ss.client, ss.user, ss.extent.getTableId().toString(), ct - ss.startTime, ct - ss.lastAccessTime, ScanType.SINGLE,
+ state, ss.extent.toThrift(), Translator.translate(ss.columnSet, Translators.CT), ss.ssiList, ss.ssio, ss.auths.getAuthorizationsBB()));
+
+ } else if (session instanceof MultiScanSession) {
+ MultiScanSession mss = (MultiScanSession) session;
+
+ ScanState state = ScanState.RUNNING;
+
+ ScanTask<MultiScanResult> nbt = mss.lookupTask;
+ if (nbt == null) {
+ state = ScanState.IDLE;
+ } else {
+ switch (nbt.getScanRunState()) {
+ case QUEUED:
+ state = ScanState.QUEUED;
+ break;
+ case FINISHED:
+ state = ScanState.IDLE;
+ break;
+ case RUNNING:
+ default:
+ /* do nothing */
+ break;
+ }
+ }
+
+ activeScans.add(new ActiveScan(mss.client, mss.user, mss.threadPoolExtent.getTableId().toString(), ct - mss.startTime, ct - mss.lastAccessTime,
+ ScanType.BATCH, state, mss.threadPoolExtent.toThrift(), Translator.translate(mss.columnSet, Translators.CT), mss.ssiList, mss.ssio, mss.auths
+ .getAuthorizationsBB()));
+ }
+ }
+
+ return activeScans;
+ }
+ }
+
+ static class TservConstraintEnv implements Environment {
+
+ private TCredentials credentials;
+ private SecurityOperation security;
+ private Authorizations auths;
+ private KeyExtent ke;
+
+ TservConstraintEnv(SecurityOperation secOp, TCredentials credentials) {
+ this.security = secOp;
+ this.credentials = credentials;
+ }
+
+ void setExtent(KeyExtent ke) {
+ this.ke = ke;
+ }
+
+ @Override
+ public KeyExtent getExtent() {
+ return ke;
+ }
+
+ @Override
+ public String getUser() {
+ return credentials.getPrincipal();
+ }
+
+ @Override
+ @Deprecated
+ public Authorizations getAuthorizations() {
+ if (auths == null)
+ try {
+ this.auths = security.getUserAuthorizations(credentials);
+ } catch (ThriftSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ return auths;
+ }
+
+ @Override
+ public AuthorizationContainer getAuthorizationsContainer() {
+ return new AuthorizationContainer() {
+ @Override
+ public boolean contains(ByteSequence auth) {
+ try {
+ return security.userHasAuthorizations(credentials,
+ Collections.<ByteBuffer> singletonList(ByteBuffer.wrap(auth.getBackingArray(), auth.offset(), auth.length())));
+ } catch (ThriftSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ }
+ }
+
+ private abstract class ScanTask<T> implements RunnableFuture<T> {
+
+ protected AtomicBoolean interruptFlag;
+ protected ArrayBlockingQueue<Object> resultQueue;
+ protected AtomicInteger state;
+ protected AtomicReference<ScanRunState> runState;
+
+ private static final int INITIAL = 1;
+ private static final int ADDED = 2;
+ private static final int CANCELED = 3;
+
+ ScanTask() {
+ interruptFlag = new AtomicBoolean(false);
+ runState = new AtomicReference<ScanRunState>(ScanRunState.QUEUED);
+ state = new AtomicInteger(INITIAL);
+ resultQueue = new ArrayBlockingQueue<Object>(1);
+ }
+
+ protected void addResult(Object o) {
+ if (state.compareAndSet(INITIAL, ADDED))
+ resultQueue.add(o);
+ else if (state.get() == ADDED)
+ throw new IllegalStateException("Tried to add more than one result");
+ }
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ if (!mayInterruptIfRunning)
+ throw new IllegalArgumentException("Cancel will always attempt to interupt running next batch task");
+
+ if (state.get() == CANCELED)
+ return true;
+
+ if (state.compareAndSet(INITIAL, CANCELED)) {
+ interruptFlag.set(true);
+ resultQueue = null;
+ return true;
+ }
+
+ return false;
+ }
+
+ @Override
+ public T get() throws InterruptedException, ExecutionException {
+ throw new UnsupportedOperationException();
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
+
+ ArrayBlockingQueue<Object> localRQ = resultQueue;
+
+ if (state.get() == CANCELED)
+ throw new CancellationException();
+
+ if (localRQ == null && state.get() == ADDED)
+ throw new IllegalStateException("Tried to get result twice");
+
+ Object r = localRQ.poll(timeout, unit);
+
+ // could have been canceled while waiting
+ if (state.get() == CANCELED) {
+ if (r != null)
+ throw new IllegalStateException("Nothing should have been added when in canceled state");
+
+ throw new CancellationException();
+ }
+
+ if (r == null)
+ throw new TimeoutException();
+
+ // make this method stop working now that something is being
+ // returned
+ resultQueue = null;
+
+ if (r instanceof Throwable)
+ throw new ExecutionException((Throwable) r);
+
+ return (T) r;
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return state.get() == CANCELED;
+ }
+
+ @Override
+ public boolean isDone() {
+ return runState.get().equals(ScanRunState.FINISHED);
+ }
+
+ public ScanRunState getScanRunState() {
+ return runState.get();
+ }
+
+ }
+
+ private static class ConditionalSession extends Session {
+ public TCredentials credentials;
+ public Authorizations auths;
+ public String tableId;
+ public AtomicBoolean interruptFlag;
+
+ @Override
+ public void cleanup() {
+ interruptFlag.set(true);
+ }
+ }
+
+ private static class UpdateSession extends Session {
+ public Tablet currentTablet;
+ public MapCounter<Tablet> successfulCommits = new MapCounter<Tablet>();
+ Map<KeyExtent,Long> failures = new HashMap<KeyExtent,Long>();
+ HashMap<KeyExtent,SecurityErrorCode> authFailures = new HashMap<KeyExtent,SecurityErrorCode>();
+ public Violations violations;
+ public TCredentials credentials;
+ public long totalUpdates = 0;
+ public long flushTime = 0;
+ Stat prepareTimes = new Stat();
+ Stat walogTimes = new Stat();
+ Stat commitTimes = new Stat();
+ Stat authTimes = new Stat();
+ public Map<Tablet,List<Mutation>> queuedMutations = new HashMap<Tablet,List<Mutation>>();
+ public long queuedMutationSize = 0;
+ TservConstraintEnv cenv = null;
+ }
+
+ private static class ScanSession extends Session {
+ public KeyExtent extent;
+ public HashSet<Column> columnSet;
+ public List<IterInfo> ssiList;
+ public Map<String,Map<String,String>> ssio;
+ public Authorizations auths;
+ public long entriesReturned = 0;
+ public Stat nbTimes = new Stat();
+ public long batchCount = 0;
+ public volatile ScanTask<ScanBatch> nextBatchTask;
+ public AtomicBoolean interruptFlag;
+ public Scanner scanner;
+ public long readaheadThreshold = Constants.SCANNER_DEFAULT_READAHEAD_THRESHOLD;
+
+ @Override
+ public void cleanup() {
+ try {
+ if (nextBatchTask != null)
+ nextBatchTask.cancel(true);
+ } finally {
+ if (scanner != null)
+ scanner.close();
+ }
+ }
+
+ }
+
+ private static class MultiScanSession extends Session {
+ HashSet<Column> columnSet;
+ Map<KeyExtent,List<Range>> queries;
+ public List<IterInfo> ssiList;
+ public Map<String,Map<String,String>> ssio;
+ public Authorizations auths;
+
+ // stats
+ int numRanges;
+ int numTablets;
+ int numEntries;
+ long totalLookupTime;
+
+ public volatile ScanTask<MultiScanResult> lookupTask;
+ public KeyExtent threadPoolExtent;
+
+ @Override
+ public void cleanup() {
+ if (lookupTask != null)
+ lookupTask.cancel(true);
+ }
+ }
+
+ /**
+ * This little class keeps track of writes in progress and allows readers to wait for writes that started before the read. It assumes that the operation ids
+ * are monotonically increasing.
- *
++ *
+ */
+ static class WriteTracker {
+ private static AtomicLong operationCounter = new AtomicLong(1);
+ private Map<TabletType,TreeSet<Long>> inProgressWrites = new EnumMap<TabletType,TreeSet<Long>>(TabletType.class);
+
+ WriteTracker() {
+ for (TabletType ttype : TabletType.values()) {
+ inProgressWrites.put(ttype, new TreeSet<Long>());
+ }
+ }
+
+ synchronized long startWrite(TabletType ttype) {
+ long operationId = operationCounter.getAndIncrement();
+ inProgressWrites.get(ttype).add(operationId);
+ return operationId;
+ }
+
+ synchronized void finishWrite(long operationId) {
+ if (operationId == -1)
+ return;
+
+ boolean removed = false;
+
+ for (TabletType ttype : TabletType.values()) {
+ removed = inProgressWrites.get(ttype).remove(operationId);
+ if (removed)
+ break;
+ }
+
+ if (!removed) {
+ throw new IllegalArgumentException("Attempted to finish write not in progress, operationId " + operationId);
+ }
+
+ this.notifyAll();
+ }
+
+ synchronized void waitForWrites(TabletType ttype) {
+ long operationId = operationCounter.getAndIncrement();
+ while (inProgressWrites.get(ttype).floor(operationId) != null) {
+ try {
+ this.wait();
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ }
+ }
+ }
+
+ public long startWrite(Set<Tablet> keySet) {
+ if (keySet.size() == 0)
+ return -1;
+
+ ArrayList<KeyExtent> extents = new ArrayList<KeyExtent>(keySet.size());
+
+ for (Tablet tablet : keySet)
+ extents.add(tablet.getExtent());
+
+ return startWrite(TabletType.type(extents));
+ }
+ }
+
+ public AccumuloConfiguration getSystemConfiguration() {
+ return serverConfig.getConfiguration();
+ }
+
+ TransactionWatcher watcher = new TransactionWatcher();
+
+ private class ThriftClientHandler extends ClientServiceHandler implements TabletClientService.Iface {
+
+ SessionManager sessionManager;
+
+ AccumuloConfiguration acuConf = getSystemConfiguration();
+
+ TabletServerUpdateMetrics updateMetrics = new TabletServerUpdateMetrics();
+
+ TabletServerScanMetrics scanMetrics = new TabletServerScanMetrics();
+
+ WriteTracker writeTracker = new WriteTracker();
+
+ private RowLocks rowLocks = new RowLocks();
+
+ ThriftClientHandler() {
+ super(instance, watcher, fs);
+ log.debug(ThriftClientHandler.class.getName() + " created");
+ sessionManager = new SessionManager(getSystemConfiguration());
+ // Register the metrics MBean
+ try {
+ updateMetrics.register();
+ scanMetrics.register();
+ } catch (Exception e) {
+ log.error("Exception registering MBean with MBean Server", e);
+ }
+ }
+
+ @Override
+ public List<TKeyExtent> bulkImport(TInfo tinfo, TCredentials credentials, long tid, Map<TKeyExtent,Map<String,MapFileInfo>> files, boolean setTime)
+ throws ThriftSecurityException {
+
+ if (!security.canPerformSystemActions(credentials))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ List<TKeyExtent> failures = new ArrayList<TKeyExtent>();
+
+ for (Entry<TKeyExtent,Map<String,MapFileInfo>> entry : files.entrySet()) {
+ TKeyExtent tke = entry.getKey();
+ Map<String,MapFileInfo> fileMap = entry.getValue();
+ Map<FileRef,MapFileInfo> fileRefMap = new HashMap<FileRef,MapFileInfo>();
+ for (Entry<String,MapFileInfo> mapping : fileMap.entrySet()) {
+ Path path = new Path(mapping.getKey());
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
+ path = ns.makeQualified(path);
+ fileRefMap.put(new FileRef(path.toString(), path), mapping.getValue());
+ }
+
+ Tablet importTablet = onlineTablets.get(new KeyExtent(tke));
+
+ if (importTablet == null) {
+ failures.add(tke);
+ } else {
+ try {
+ importTablet.importMapFiles(tid, fileRefMap, setTime);
+ } catch (IOException ioe) {
+ log.info("files " + fileMap.keySet() + " not imported to " + new KeyExtent(tke) + ": " + ioe.getMessage());
+ failures.add(tke);
+ }
+ }
+ }
+ return failures;
+ }
+
+ private class NextBatchTask extends ScanTask<ScanBatch> {
+
+ private long scanID;
+
+ NextBatchTask(long scanID, AtomicBoolean interruptFlag) {
+ this.scanID = scanID;
+ this.interruptFlag = interruptFlag;
+
+ if (interruptFlag.get())
+ cancel(true);
+ }
+
+ @Override
+ public void run() {
+
+ final ScanSession scanSession = (ScanSession) sessionManager.getSession(scanID);
+ String oldThreadName = Thread.currentThread().getName();
+
+ try {
+ if (isCancelled() || scanSession == null)
+ return;
+
+ runState.set(ScanRunState.RUNNING);
+
+ Thread.currentThread().setName(
+ "User: " + scanSession.user + " Start: " + scanSession.startTime + " Client: " + scanSession.client + " Tablet: " + scanSession.extent);
+
+ Tablet tablet = onlineTablets.get(scanSession.extent);
+
+ if (tablet == null) {
+ addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
+ return;
+ }
+
+ long t1 = System.currentTimeMillis();
+ ScanBatch batch = scanSession.scanner.read();
+ long t2 = System.currentTimeMillis();
+ scanSession.nbTimes.addStat(t2 - t1);
+
+ // there should only be one thing on the queue at a time, so
+ // it should be ok to call add()
+ // instead of put()... if add() fails because queue is at
+ // capacity it means there is code
+ // problem somewhere
+ addResult(batch);
+ } catch (TabletClosedException e) {
+ addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
+ } catch (IterationInterruptedException iie) {
+ if (!isCancelled()) {
+ log.warn("Iteration interrupted, when scan not cancelled", iie);
+ addResult(iie);
+ }
+ } catch (TooManyFilesException tmfe) {
+ addResult(tmfe);
+ } catch (Throwable e) {
+ log.warn("exception while scanning tablet " + (scanSession == null ? "(unknown)" : scanSession.extent), e);
+ addResult(e);
+ } finally {
+ runState.set(ScanRunState.FINISHED);
+ Thread.currentThread().setName(oldThreadName);
+ }
+
+ }
+ }
+
+ private class LookupTask extends ScanTask<MultiScanResult> {
+
+ private long scanID;
+
+ LookupTask(long scanID) {
+ this.scanID = scanID;
+ }
+
+ @Override
+ public void run() {
+ MultiScanSession session = (MultiScanSession) sessionManager.getSession(scanID);
+ String oldThreadName = Thread.currentThread().getName();
+
+ try {
+ if (isCancelled() || session == null)
+ return;
+
+ TableConfiguration acuTableConf = ServerConfiguration.getTableConfiguration(instance, session.threadPoolExtent.getTableId().toString());
+ long maxResultsSize = acuTableConf.getMemoryInBytes(Property.TABLE_SCAN_MAXMEM);
+
+ runState.set(ScanRunState.RUNNING);
+ Thread.currentThread().setName("Client: " + session.client + " User: " + session.user + " Start: " + session.startTime + " Table: ");
+
+ long bytesAdded = 0;
+ long maxScanTime = 4000;
+
+ long startTime = System.currentTimeMillis();
+
+ ArrayList<KVEntry> results = new ArrayList<KVEntry>();
+ Map<KeyExtent,List<Range>> failures = new HashMap<KeyExtent,List<Range>>();
+ ArrayList<KeyExtent> fullScans = new ArrayList<KeyExtent>();
+ KeyExtent partScan = null;
+ Key partNextKey = null;
+ boolean partNextKeyInclusive = false;
+
+ Iterator<Entry<KeyExtent,List<Range>>> iter = session.queries.entrySet().iterator();
+
+ // check the time so that the read ahead thread is not monopolized
+ while (iter.hasNext() && bytesAdded < maxResultsSize && (System.currentTimeMillis() - startTime) < maxScanTime) {
+ Entry<KeyExtent,List<Range>> entry = iter.next();
+
+ iter.remove();
+
+ // check that tablet server is serving requested tablet
+ Tablet tablet = onlineTablets.get(entry.getKey());
+ if (tablet == null) {
+ failures.put(entry.getKey(), entry.getValue());
+ continue;
+ }
+ Thread.currentThread().setName(
+ "Client: " + session.client + " User: " + session.user + " Start: " + session.startTime + " Tablet: " + entry.getKey().toString());
+
+ LookupResult lookupResult;
+ try {
+
+ // do the following check to avoid a race condition
+ // between setting false below and the task being
+ // canceled
+ if (isCancelled())
+ interruptFlag.set(true);
+
+ lookupResult = tablet.lookup(entry.getValue(), session.columnSet, session.auths, results, maxResultsSize - bytesAdded, session.ssiList,
+ session.ssio, interruptFlag);
+
+ // if the tablet was closed it it possible that the
+ // interrupt flag was set.... do not want it set for
+ // the next
+ // lookup
+ interruptFlag.set(false);
+
+ } catch (IOException e) {
+ log.warn("lookup failed for tablet " + entry.getKey(), e);
+ throw new RuntimeException(e);
+ }
+
+ bytesAdded += lookupResult.bytesAdded;
+
+ if (lookupResult.unfinishedRanges.size() > 0) {
+ if (lookupResult.closed) {
+ failures.put(entry.getKey(), lookupResult.unfinishedRanges);
+ } else {
+ session.queries.put(entry.getKey(), lookupResult.unfinishedRanges);
+ partScan = entry.getKey();
+ partNextKey = lookupResult.unfinishedRanges.get(0).getStartKey();
+ partNextKeyInclusive = lookupResult.unfinishedRanges.get(0).isStartKeyInclusive();
+ }
+ } else {
+ fullScans.add(entry.getKey());
+ }
+ }
+
+ long finishTime = System.currentTimeMillis();
+ session.totalLookupTime += (finishTime - startTime);
+ session.numEntries += results.size();
+
+ // convert everything to thrift before adding result
+ List<TKeyValue> retResults = new ArrayList<TKeyValue>();
+ for (KVEntry entry : results)
+ retResults.add(new TKeyValue(entry.key.toThrift(), ByteBuffer.wrap(entry.value)));
+ Map<TKeyExtent,List<TRange>> retFailures = Translator.translate(failures, Translators.KET, new Translator.ListTranslator<Range,TRange>(Translators.RT));
+ List<TKeyExtent> retFullScans = Translator.translate(fullScans, Translators.KET);
+ TKeyExtent retPartScan = null;
+ TKey retPartNextKey = null;
+ if (partScan != null) {
+ retPartScan = partScan.toThrift();
+ retPartNextKey = partNextKey.toThrift();
+ }
+ // add results to queue
+ addResult(new MultiScanResult(retResults, retFailures, retFullScans, retPartScan, retPartNextKey, partNextKeyInclusive, session.queries.size() != 0));
+ } catch (IterationInterruptedException iie) {
+ if (!isCancelled()) {
+ log.warn("Iteration interrupted, when scan not cancelled", iie);
+ addResult(iie);
+ }
+ } catch (Throwable e) {
+ log.warn("exception while doing multi-scan ", e);
+ addResult(e);
+ } finally {
+ Thread.currentThread().setName(oldThreadName);
+ runState.set(ScanRunState.FINISHED);
+ }
+ }
+ }
+
+ @Override
+ public InitialScan startScan(TInfo tinfo, TCredentials credentials, TKeyExtent textent, TRange range, List<TColumn> columns, int batchSize,
+ List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites, boolean isolated,
+ long readaheadThreshold) throws NotServingTabletException, ThriftSecurityException, org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
+
+ String tableId = new String(textent.getTable(), Constants.UTF8);
+ if (!security.canScan(credentials, tableId, Tables.getNamespaceId(instance, tableId), range, columns, ssiList, ssio, authorizations))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ if (!security.userHasAuthorizations(credentials, authorizations))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
+
+ KeyExtent extent = new KeyExtent(textent);
+
+ // wait for any writes that are in flight.. this done to ensure
+ // consistency across client restarts... assume a client writes
+ // to accumulo and dies while waiting for a confirmation from
+ // accumulo... the client process restarts and tries to read
+ // data from accumulo making the assumption that it will get
+ // any writes previously made... however if the server side thread
+ // processing the write from the dead client is still in progress,
+ // the restarted client may not see the write unless we wait here.
+ // this behavior is very important when the client is reading the
+ // metadata
+ if (waitForWrites)
+ writeTracker.waitForWrites(TabletType.type(extent));
+
+ Tablet tablet = onlineTablets.get(extent);
+ if (tablet == null)
+ throw new NotServingTabletException(textent);
+
+ ScanSession scanSession = new ScanSession();
+ scanSession.user = credentials.getPrincipal();
+ scanSession.extent = new KeyExtent(extent);
+ scanSession.columnSet = new HashSet<Column>();
+ scanSession.ssiList = ssiList;
+ scanSession.ssio = ssio;
+ scanSession.auths = new Authorizations(authorizations);
+ scanSession.interruptFlag = new AtomicBoolean();
+ scanSession.readaheadThreshold = readaheadThreshold;
+
+ for (TColumn tcolumn : columns) {
+ scanSession.columnSet.add(new Column(tcolumn));
+ }
+
+ scanSession.scanner = tablet.createScanner(new Range(range), batchSize, scanSession.columnSet, scanSession.auths, ssiList, ssio, isolated,
+ scanSession.interruptFlag);
+
+ long sid = sessionManager.createSession(scanSession, true);
+
+ ScanResult scanResult;
+ try {
+ scanResult = continueScan(tinfo, sid, scanSession);
+ } catch (NoSuchScanIDException e) {
+ log.error("The impossible happened", e);
+ throw new RuntimeException();
+ } finally {
+ sessionManager.unreserveSession(sid);
+ }
+
+ return new InitialScan(sid, scanResult);
+ }
+
+ @Override
+ public ScanResult continueScan(TInfo tinfo, long scanID) throws NoSuchScanIDException, NotServingTabletException,
+ org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
+ ScanSession scanSession = (ScanSession) sessionManager.reserveSession(scanID);
+ if (scanSession == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ try {
+ return continueScan(tinfo, scanID, scanSession);
+ } finally {
+ sessionManager.unreserveSession(scanSession);
+ }
+ }
+
+ private ScanResult continueScan(TInfo tinfo, long scanID, ScanSession scanSession) throws NoSuchScanIDException, NotServingTabletException,
+ org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
+
+ if (scanSession.nextBatchTask == null) {
+ scanSession.nextBatchTask = new NextBatchTask(scanID, scanSession.interruptFlag);
+ resourceManager.executeReadAhead(scanSession.extent, scanSession.nextBatchTask);
+ }
+
+ ScanBatch bresult;
+ try {
+ bresult = scanSession.nextBatchTask.get(MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS, TimeUnit.MILLISECONDS);
+ scanSession.nextBatchTask = null;
+ } catch (ExecutionException e) {
+ sessionManager.removeSession(scanID);
+ if (e.getCause() instanceof NotServingTabletException)
+ throw (NotServingTabletException) e.getCause();
+ else if (e.getCause() instanceof TooManyFilesException)
+ throw new org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException(scanSession.extent.toThrift());
+ else
+ throw new RuntimeException(e);
+ } catch (CancellationException ce) {
+ sessionManager.removeSession(scanID);
+ Tablet tablet = onlineTablets.get(scanSession.extent);
+ if (tablet == null || tablet.isClosed())
+ throw new NotServingTabletException(scanSession.extent.toThrift());
+ else
+ throw new NoSuchScanIDException();
+ } catch (TimeoutException e) {
+ List<TKeyValue> param = Collections.emptyList();
+ long timeout = acuConf.getTimeInMillis(Property.TSERV_CLIENT_TIMEOUT);
+ sessionManager.removeIfNotAccessed(scanID, timeout);
+ return new ScanResult(param, true);
+ } catch (Throwable t) {
+ sessionManager.removeSession(scanID);
+ log.warn("Failed to get next batch", t);
+ throw new RuntimeException(t);
+ }
+
+ ScanResult scanResult = new ScanResult(Key.compress(bresult.results), bresult.more);
+
+ scanSession.entriesReturned += scanResult.results.size();
+
+ scanSession.batchCount++;
+
+ if (scanResult.more && scanSession.batchCount > scanSession.readaheadThreshold) {
+ // start reading next batch while current batch is transmitted
+ // to client
+ scanSession.nextBatchTask = new NextBatchTask(scanID, scanSession.interruptFlag);
+ resourceManager.executeReadAhead(scanSession.extent, scanSession.nextBatchTask);
+ }
+
+ if (!scanResult.more)
+ closeScan(tinfo, scanID);
+
+ return scanResult;
+ }
+
+ @Override
+ public void closeScan(TInfo tinfo, long scanID) {
+ ScanSession ss = (ScanSession) sessionManager.removeSession(scanID);
+ if (ss != null) {
+ long t2 = System.currentTimeMillis();
+
+ log.debug(String.format("ScanSess tid %s %s %,d entries in %.2f secs, nbTimes = [%s] ", TServerUtils.clientAddress.get(), ss.extent.getTableId()
+ .toString(), ss.entriesReturned, (t2 - ss.startTime) / 1000.0, ss.nbTimes.toString()));
+ if (scanMetrics.isEnabled()) {
+ scanMetrics.add(TabletServerScanMetrics.scan, t2 - ss.startTime);
+ scanMetrics.add(TabletServerScanMetrics.resultSize, ss.entriesReturned);
+ }
+ }
+ }
+
+ @Override
+ public InitialMultiScan startMultiScan(TInfo tinfo, TCredentials credentials, Map<TKeyExtent,List<TRange>> tbatch, List<TColumn> tcolumns,
+ List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites) throws ThriftSecurityException {
+ // find all of the tables that need to be scanned
+ HashSet<String> tables = new HashSet<String>();
+ for (TKeyExtent keyExtent : tbatch.keySet()) {
+ tables.add(new String(keyExtent.getTable(), Constants.UTF8));
+ }
+
+ if (tables.size() != 1)
+ throw new IllegalArgumentException("Cannot batch scan over multiple tables");
+
+ // check if user has permission to the tables
+ for (String tableId : tables)
+ if (!security.canScan(credentials, tableId, Tables.getNamespaceId(instance, tableId), tbatch, tcolumns, ssiList, ssio, authorizations))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ try {
+ if (!security.userHasAuthorizations(credentials, authorizations))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
+ } catch (ThriftSecurityException tse) {
+ log.error(tse, tse);
+ throw tse;
+ }
+ Map<KeyExtent,List<Range>> batch = Translator.translate(tbatch, new TKeyExtentTranslator(), new Translator.ListTranslator<TRange,Range>(
+ new TRangeTranslator()));
+
+ // This is used to determine which thread pool to use
+ KeyExtent threadPoolExtent = batch.keySet().iterator().next();
+
+ if (waitForWrites)
+ writeTracker.waitForWrites(TabletType.type(batch.keySet()));
+
+ MultiScanSession mss = new MultiScanSession();
+ mss.user = credentials.getPrincipal();
+ mss.queries = batch;
+ mss.columnSet = new HashSet<Column>(tcolumns.size());
+ mss.ssiList = ssiList;
+ mss.ssio = ssio;
+ mss.auths = new Authorizations(authorizations);
+
+ mss.numTablets = batch.size();
+ for (List<Range> ranges : batch.values()) {
+ mss.numRanges += ranges.size();
+ }
+
+ for (TColumn tcolumn : tcolumns)
+ mss.columnSet.add(new Column(tcolumn));
+
+ mss.threadPoolExtent = threadPoolExtent;
+
+ long sid = sessionManager.createSession(mss, true);
+
+ MultiScanResult result;
+ try {
+ result = continueMultiScan(tinfo, sid, mss);
+ } catch (NoSuchScanIDException e) {
+ log.error("the impossible happened", e);
+ throw new RuntimeException("the impossible happened", e);
+ } finally {
+ sessionManager.unreserveSession(sid);
+ }
+
+ return new InitialMultiScan(sid, result);
+ }
+
+ @Override
+ public MultiScanResult continueMultiScan(TInfo tinfo, long scanID) throws NoSuchScanIDException {
+
+ MultiScanSession session = (MultiScanSession) sessionManager.reserveSession(scanID);
+
+ if (session == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ try {
+ return continueMultiScan(tinfo, scanID, session);
+ } finally {
+ sessionManager.unreserveSession(session);
+ }
+ }
+
+ private MultiScanResult continueMultiScan(TInfo tinfo, long scanID, MultiScanSession session) throws NoSuchScanIDException {
+
+ if (session.lookupTask == null) {
+ session.lookupTask = new LookupTask(scanID);
+ resourceManager.executeReadAhead(session.threadPoolExtent, session.lookupTask);
+ }
+
+ try {
+ MultiScanResult scanResult = session.lookupTask.get(MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS, TimeUnit.MILLISECONDS);
+ session.lookupTask = null;
+ return scanResult;
+ } catch (TimeoutException e1) {
+ long timeout = acuConf.getTimeInMillis(Property.TSERV_CLIENT_TIMEOUT);
+ sessionManager.removeIfNotAccessed(scanID, timeout);
+ List<TKeyValue> results = Collections.emptyList();
+ Map<TKeyExtent,List<TRange>> failures = Collections.emptyMap();
+ List<TKeyExtent> fullScans = Collections.emptyList();
+ return new MultiScanResult(results, failures, fullScans, null, null, false, true);
+ } catch (Throwable t) {
+ sessionManager.removeSession(scanID);
+ log.warn("Failed to get multiscan result", t);
+ throw new RuntimeException(t);
+ }
+ }
+
+ @Override
+ public void closeMultiScan(TInfo tinfo, long scanID) throws NoSuchScanIDException {
+ MultiScanSession session = (MultiScanSession) sessionManager.removeSession(scanID);
+ if (session == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ long t2 = System.currentTimeMillis();
+ log.debug(String.format("MultiScanSess %s %,d entries in %.2f secs (lookup_time:%.2f secs tablets:%,d ranges:%,d) ", TServerUtils.clientAddress.get(),
+ session.numEntries, (t2 - session.startTime) / 1000.0, session.totalLookupTime / 1000.0, session.numTablets, session.numRanges));
+ }
+
+ @Override
+ public long startUpdate(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException {
+ // Make sure user is real
+
+ security.authenticateUser(credentials, credentials);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
+
+ UpdateSession us = new UpdateSession();
+ us.violations = new Violations();
+ us.credentials = credentials;
+ us.cenv = new TservConstraintEnv(security, us.credentials);
+
+ long sid = sessionManager.createSession(us, false);
+
+ return sid;
+ }
+
+ private void setUpdateTablet(UpdateSession us, KeyExtent keyExtent) {
+ long t1 = System.currentTimeMillis();
+ if (us.currentTablet != null && us.currentTablet.getExtent().equals(keyExtent))
+ return;
+ if (us.currentTablet == null && (us.failures.containsKey(keyExtent) || us.authFailures.containsKey(keyExtent))) {
+ // if there were previous failures, then do not accept additional writes
+ return;
+ }
+
+ try {
+ // if user has no permission to write to this table, add it to
+ // the failures list
+ boolean sameTable = us.currentTablet != null && (us.currentTablet.getExtent().getTableId().equals(keyExtent.getTableId()));
+ String tableId = keyExtent.getTableId().toString();
+ if (sameTable || security.canWrite(us.credentials, tableId, Tables.getNamespaceId(instance, tableId))) {
+ long t2 = System.currentTimeMillis();
+ us.authTimes.addStat(t2 - t1);
+ us.currentTablet = onlineTablets.get(keyExtent);
+ if (us.currentTablet != null) {
+ us.queuedMutations.put(us.currentTablet, new ArrayList<Mutation>());
+ } else {
+ // not serving tablet, so report all mutations as
+ // failures
+ us.failures.put(keyExtent, 0l);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.unknownTabletErrors, 0);
+ }
+ } else {
+ log.warn("Denying access to table " + keyExtent.getTableId() + " for user " + us.credentials.getPrincipal());
+ long t2 = System.currentTimeMillis();
+ us.authTimes.addStat(t2 - t1);
+ us.currentTablet = null;
+ us.authFailures.put(keyExtent, SecurityErrorCode.PERMISSION_DENIED);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
+ return;
+ }
+ } catch (ThriftSecurityException e) {
+ log.error("Denying permission to check user " + us.credentials.getPrincipal() + " with user " + e.getUser(), e);
+ long t2 = System.currentTimeMillis();
+ us.authTimes.addStat(t2 - t1);
+ us.currentTablet = null;
+ us.authFailures.put(keyExtent, e.getCode());
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
+ return;
+ }
+ }
+
+ @Override
+ public void applyUpdates(TInfo tinfo, long updateID, TKeyExtent tkeyExtent, List<TMutation> tmutations) {
+ UpdateSession us = (UpdateSession) sessionManager.reserveSession(updateID);
+ if (us == null) {
+ throw new RuntimeException("No Such SessionID");
+ }
+
+ try {
+ KeyExtent keyExtent = new KeyExtent(tkeyExtent);
+ setUpdateTablet(us, keyExtent);
+
+ if (us.currentTablet != null) {
+ List<Mutation> mutations = us.queuedMutations.get(us.currentTablet);
+ for (TMutation tmutation : tmutations) {
+ Mutation mutation = new ServerMutation(tmutation);
+ mutations.add(mutation);
+ us.queuedMutationSize += mutation.numBytes();
+ }
+ if (us.queuedMutationSize > getSystemConfiguration().getMemoryInBytes(Property.TSERV_MUTATION_QUEUE_MAX))
+ flush(us);
+ }
+ } finally {
+ sessionManager.unreserveSession(us);
+ }
+ }
+
+ private void flush(UpdateSession us) {
+
+ int mutationCount = 0;
+ Map<CommitSession,List<Mutation>> sendables = new HashMap<CommitSession,List<Mutation>>();
+ Throwable error = null;
+
+ long pt1 = System.currentTimeMillis();
+
+ boolean containsMetadataTablet = false;
+ for (Tablet tablet : us.queuedMutations.keySet())
+ if (tablet.getExtent().isMeta())
+ containsMetadataTablet = true;
+
+ if (!containsMetadataTablet && us.queuedMutations.size() > 0)
+ TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
+
+ Span prep = Trace.start("prep");
+ try {
+ for (Entry<Tablet,? extends List<Mutation>> entry : us.queuedMutations.entrySet()) {
+
+ Tablet tablet = entry.getKey();
+ List<Mutation> mutations = entry.getValue();
+ if (mutations.size() > 0) {
+ try {
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.mutationArraySize, mutations.size());
+
+ CommitSession commitSession = tablet.prepareMutationsForCommit(us.cenv, mutations);
+ if (commitSession == null) {
+ if (us.currentTablet == tablet) {
+ us.currentTablet = null;
+ }
+ us.failures.put(tablet.getExtent(), us.successfulCommits.get(tablet));
+ } else {
+ sendables.put(commitSession, mutations);
+ mutationCount += mutations.size();
+ }
+
+ } catch (TConstraintViolationException e) {
+ us.violations.add(e.getViolations());
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.constraintViolations, 0);
+
+ if (e.getNonViolators().size() > 0) {
+ // only log and commit mutations if there were some
+ // that did not
+ // violate constraints... this is what
+ // prepareMutationsForCommit()
+ // expects
+ sendables.put(e.getCommitSession(), e.getNonViolators());
+ }
+
+ mutationCount += mutations.size();
+
+ } catch (HoldTimeoutException t) {
+ error = t;
+ log.debug("Giving up on mutations due to a long memory hold time");
+ break;
+ } catch (Throwable t) {
+ error = t;
+ log.error("Unexpected error preparing for commit", error);
+ break;
+ }
+ }
+ }
+ } finally {
+ prep.stop();
+ }
+
+ long pt2 = System.currentTimeMillis();
+ us.prepareTimes.addStat(pt2 - pt1);
+ updateAvgPrepTime(pt2 - pt1, us.queuedMutations.size());
+
+ if (error != null) {
+ for (Entry<CommitSession,List<Mutation>> e : sendables.entrySet()) {
+ e.getKey().abortCommit(e.getValue());
+ }
+ throw new RuntimeException(error);
+ }
+ try {
+ Span wal = Trace.start("wal");
+ try {
+ while (true) {
+ try {
+ long t1 = System.currentTimeMillis();
+
+ logger.logManyTablets(sendables);
+
+ long t2 = System.currentTimeMillis();
+ us.walogTimes.addStat(t2 - t1);
+ updateWalogWriteTime((t2 - t1));
+ break;
+ } catch (IOException ex) {
+ log.warn("logging mutations failed, retrying");
+ } catch (FSError ex) { // happens when DFS is localFS
+ log.warn("logging mutations failed, retrying");
+ } catch (Throwable t) {
+ log.error("Unknown exception logging mutations, counts for mutations in flight not decremented!", t);
+ throw new RuntimeException(t);
+ }
+ }
+ } finally {
+ wal.stop();
+ }
+
+ Span commit = Trace.start("commit");
+ try {
+ long t1 = System.currentTimeMillis();
+ for (Entry<CommitSession,? extends List<Mutation>> entry : sendables.entrySet()) {
+ CommitSession commitSession = entry.getKey();
+ List<Mutation> mutations = entry.getValue();
+
+ commitSession.commit(mutations);
+
+ Tablet tablet = commitSession.getTablet();
+
+ if (tablet == us.currentTablet) {
+ // because constraint violations may filter out some
+ // mutations, for proper
+ // accounting with the client code, need to increment
+ // the count based
+ // on the original number of mutations from the client
+ // NOT the filtered number
+ us.successfulCommits.increment(tablet, us.queuedMutations.get(tablet).size());
+ }
+ }
+ long t2 = System.currentTimeMillis();
+
+ us.flushTime += (t2 - pt1);
+ us.commitTimes.addStat(t2 - t1);
+
+ updateAvgCommitTime(t2 - t1, sendables.size());
+ } finally {
+ commit.stop();
+ }
+ } finally {
+ us.queuedMutations.clear();
+ if (us.currentTablet != null) {
+ us.queuedMutations.put(us.currentTablet, new ArrayList<Mutation>());
+ }
+ us.queuedMutationSize = 0;
+ }
+ us.totalUpdates += mutationCount;
+ }
+
+ private void updateWalogWriteTime(long time) {
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.waLogWriteTime, time);
+ }
+
+ private void updateAvgCommitTime(long time, int size) {
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.commitTime, (long) ((time) / (double) size));
+ }
+
+ private void updateAvgPrepTime(long time, int size) {
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.commitPrep, (long) ((time) / (double) size));
+ }
+
+ @Override
+ public UpdateErrors closeUpdate(TInfo tinfo, long updateID) throws NoSuchScanIDException {
+ UpdateSession us = (UpdateSession) sessionManager.removeSession(updateID);
+ if (us == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ // clients may or may not see data from an update session while
+ // it is in progress, however when the update session is closed
+ // want to ensure that reads wait for the write to finish
+ long opid = writeTracker.startWrite(us.queuedMutations.keySet());
+
+ try {
+ flush(us);
+ } finally {
+ writeTracker.finishWrite(opid);
+ }
+
+ log.debug(String.format("UpSess %s %,d in %.3fs, at=[%s] ft=%.3fs(pt=%.3fs lt=%.3fs ct=%.3fs)", TServerUtils.clientAddress.get(), us.totalUpdates,
+ (System.currentTimeMillis() - us.startTime) / 1000.0, us.authTimes.toString(), us.flushTime / 1000.0, us.prepareTimes.getSum() / 1000.0,
+ us.walogTimes.getSum() / 1000.0, us.commitTimes.getSum() / 1000.0));
+ if (us.failures.size() > 0) {
+ Entry<KeyExtent,Long> first = us.failures.entrySet().iterator().next();
+ log.debug(String.format("Failures: %d, first extent %s successful commits: %d", us.failures.size(), first.getKey().toString(), first.getValue()));
+ }
+ List<ConstraintViolationSummary> violations = us.violations.asList();
+ if (violations.size() > 0) {
+ ConstraintViolationSummary first = us.violations.asList().iterator().next();
+ log.debug(String.format("Violations: %d, first %s occurs %d", violations.size(), first.violationDescription, first.numberOfViolatingMutations));
+ }
+ if (us.authFailures.size() > 0) {
+ KeyExtent first = us.authFailures.keySet().iterator().next();
+ log.debug(String.format("Authentication Failures: %d, first %s", us.authFailures.size(), first.toString()));
+ }
+
+ return new UpdateErrors(Translator.translate(us.failures, Translators.KET), Translator.translate(violations, Translators.CVST), Translator.translate(
+ us.authFailures, Translators.KET));
+ }
+
+ @Override
+ public void update(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, TMutation tmutation) throws NotServingTabletException,
+ ConstraintViolationException, ThriftSecurityException {
+
+ String tableId = new String(tkeyExtent.getTable(), Constants.UTF8);
+ if (!security.canWrite(credentials, tableId, Tables.getNamespaceId(instance, tableId)))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ KeyExtent keyExtent = new KeyExtent(tkeyExtent);
+ Tablet tablet = onlineTablets.get(new KeyExtent(keyExtent));
+ if (tablet == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+
+ if (!keyExtent.isMeta())
+ TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
+
+ long opid = writeTracker.startWrite(TabletType.type(keyExtent));
+
+ try {
+ Mutation mutation = new ServerMutation(tmutation);
+ List<Mutation> mutations = Collections.singletonList(mutation);
+
+ Span prep = Trace.start("prep");
+ CommitSession cs;
+ try {
+ cs = tablet.prepareMutationsForCommit(new TservConstraintEnv(security, credentials), mutations);
+ } finally {
+ prep.stop();
+ }
+ if (cs == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+
+ while (true) {
+ try {
+ Span wal = Trace.start("wal");
+ try {
+ logger.log(cs, cs.getWALogSeq(), mutation);
+ } finally {
+ wal.stop();
+ }
+ break;
+ } catch (IOException ex) {
+ log.warn(ex, ex);
+ }
+ }
+
+ Span commit = Trace.start("commit");
+ try {
+ cs.commit(mutations);
+ } finally {
+ commit.stop();
+ }
+ } catch (TConstraintViolationException e) {
+ throw new ConstraintViolationException(Translator.translate(e.getViolations().asList(), Translators.CVST));
+ } finally {
+ writeTracker.finishWrite(opid);
+ }
+ }
+
+ private void checkConditions(Map<KeyExtent,List<ServerConditionalMutation>> updates, ArrayList<TCMResult> results, ConditionalSession cs,
+ List<String> symbols) throws IOException {
+ Iterator<Entry<KeyExtent,List<ServerConditionalMutation>>> iter = updates.entrySet().iterator();
+
+ CompressedIterators compressedIters = new CompressedIterators(symbols);
+
+ while (iter.hasNext()) {
+ Entry<KeyExtent,List<ServerConditionalMutation>> entry = iter.next();
+ Tablet tablet = onlineTablets.get(entry.getKey());
+
+ if (tablet == null || tablet.isClosed()) {
+ for (ServerConditionalMutation scm : entry.getValue())
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ iter.remove();
+ } else {
+ List<ServerConditionalMutation> okMutations = new ArrayList<ServerConditionalMutation>(entry.getValue().size());
+
+ for (ServerConditionalMutation scm : entry.getValue()) {
+ if (checkCondition(results, cs, compressedIters, tablet, scm))
+ okMutations.add(scm);
+ }
+
+ entry.setValue(okMutations);
+ }
+
+ }
+ }
+
+ boolean checkCondition(ArrayList<TCMResult> results, ConditionalSession cs, CompressedIterators compressedIters, Tablet tablet,
+ ServerConditionalMutation scm) throws IOException {
+ boolean add = true;
+
+ Set<Column> emptyCols = Collections.emptySet();
+
+ for (TCondition tc : scm.getConditions()) {
+
+ Range range;
+ if (tc.hasTimestamp)
+ range = Range.exact(new Text(scm.getRow()), new Text(tc.getCf()), new Text(tc.getCq()), new Text(tc.getCv()), tc.getTs());
+ else
+ range = Range.exact(new Text(scm.getRow()), new Text(tc.getCf()), new Text(tc.getCq()), new Text(tc.getCv()));
+
+ IterConfig ic = compressedIters.decompress(tc.iterators);
+
+ Scanner scanner = tablet.createScanner(range, 1, emptyCols, cs.auths, ic.ssiList, ic.ssio, false, cs.interruptFlag);
+
+ try {
+ ScanBatch batch = scanner.read();
+
+ Value val = null;
+
+ for (KVEntry entry2 : batch.results) {
+ val = entry2.getValue();
+ break;
+ }
+
+ if ((val == null ^ tc.getVal() == null) || (val != null && !Arrays.equals(tc.getVal(), val.get()))) {
+ results.add(new TCMResult(scm.getID(), TCMStatus.REJECTED));
+ add = false;
+ break;
+ }
+
+ } catch (TabletClosedException e) {
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ add = false;
+ break;
+ } catch (IterationInterruptedException iie) {
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ add = false;
+ break;
+ } catch (TooManyFilesException tmfe) {
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ add = false;
+ break;
+ }
+ }
+ return add;
+ }
+
+ private void writeConditionalMutations(Map<KeyExtent,List<ServerConditionalMutation>> updates, ArrayList<TCMResult> results, ConditionalSession sess) {
+ Set<Entry<KeyExtent,List<ServerConditionalMutation>>> es = updates.entrySet();
+
+ Map<CommitSession,List<Mutation>> sendables = new HashMap<CommitSession,List<Mutation>>();
+
+ boolean sessionCanceled = sess.interruptFlag.get();
+
+ Span prepSpan = Trace.start("prep");
+ try {
+ long t1 = System.currentTimeMillis();
+ for (Entry<KeyExtent,List<ServerConditionalMutation>> entry : es) {
+ Tablet tablet = onlineTablets.get(entry.getKey());
+ if (tablet == null || tablet.isClosed() || sessionCanceled) {
+ for (ServerConditionalMutation scm : entry.getValue())
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ } else {
+ try {
+
+ @SuppressWarnings("unchecked")
+ List<Mutation> mutations = (List<Mutation>) (List<? extends Mutation>) entry.getValue();
+ if (mutations.size() > 0) {
+
+ CommitSession cs = tablet.prepareMutationsForCommit(new TservConstraintEnv(security, sess.credentials), mutations);
+
+ if (cs == null) {
+ for (ServerConditionalMutation scm : entry.getValue())
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ } else {
+ for (ServerConditionalMutation scm : entry.getValue())
+ results.add(new TCMResult(scm.getID(), TCMStatus.ACCEPTED));
+ sendables.put(cs, mutations);
+ }
+ }
+ } catch (TConstraintViolationException e) {
+ if (e.getNonViolators().size() > 0) {
+ sendables.put(e.getCommitSession(), e.getNonViolators());
+ for (Mutation m : e.getNonViolators())
+ results.add(new TCMResult(((ServerConditionalMutation) m).getID(), TCMStatus.ACCEPTED));
+ }
+
+ for (Mutation m : e.getViolators())
+ results.add(new TCMResult(((ServerConditionalMutation) m).getID(), TCMStatus.VIOLATED));
+ }
+ }
+ }
+
+ long t2 = System.currentTimeMillis();
+ updateAvgPrepTime(t2 - t1, es.size());
+ } finally {
+ prepSpan.stop();
+ }
+
+ Span walSpan = Trace.start("wal");
+ try {
+ while (true && sendables.size() > 0) {
+ try {
+ long t1 = System.currentTimeMillis();
+ logger.logManyTablets(sendables);
+ long t2 = System.currentTimeMillis();
+ updateWalogWriteTime(t2 - t1);
+ break;
+ } catch (IOException ex) {
+ log.warn("logging mutations failed, retrying");
+ } catch (FSError ex) { // happens when DFS is localFS
+ log.warn("logging mutations failed, retrying");
+ } catch (Throwable t) {
+ log.error("Unknown exception logging mutations, counts for mutations in flight not decremented!", t);
+ throw new RuntimeException(t);
+ }
+ }
+ } finally {
+ walSpan.stop();
+ }
+
+ Span commitSpan = Trace.start("commit");
+ try {
+ long t1 = System.currentTimeMillis();
+ for (Entry<CommitSession,? extends List<Mutation>> entry : sendables.entrySet()) {
+ CommitSession commitSession = entry.getKey();
+ List<Mutation> mutations = entry.getValue();
+
+ commitSession.commit(mutations);
+ }
+ long t2 = System.currentTimeMillis();
+ updateAvgCommitTime(t2 - t1, sendables.size());
+ } finally {
+ commitSpan.stop();
+ }
+
+ }
+
+ private Map<KeyExtent,List<ServerConditionalMutation>> conditionalUpdate(ConditionalSession cs, Map<KeyExtent,List<ServerConditionalMutation>> updates,
+ ArrayList<TCMResult> results, List<String> symbols) throws IOException {
+ // sort each list of mutations, this is done to avoid deadlock and doing seeks in order is more efficient and detect duplicate rows.
+ ConditionalMutationSet.sortConditionalMutations(updates);
+
+ Map<KeyExtent,List<ServerConditionalMutation>> deferred = new HashMap<KeyExtent,List<ServerConditionalMutation>>();
+
+ // can not process two mutations for the same row, because one will not see what the other writes
+ ConditionalMutationSet.deferDuplicatesRows(updates, deferred);
+
+ // get as many locks as possible w/o blocking... defer any rows that are locked
+ List<RowLock> locks = rowLocks.acquireRowlocks(updates, deferred);
+ try {
+ Span checkSpan = Trace.start("Check conditions");
+ try {
+ checkConditions(updates, results, cs, symbols);
+ } finally {
+ checkSpan.stop();
+ }
+
+ Span updateSpan = Trace.start("apply conditional mutations");
+ try {
+ writeConditionalMutations(updates, results, cs);
+ } finally {
+ updateSpan.stop();
+ }
+ } finally {
+ rowLocks.releaseRowLocks(locks);
+ }
+ return deferred;
+ }
+
+ @Override
+ public TConditionalSession startConditionalUpdate(TInfo tinfo, TCredentials credentials, List<ByteBuffer> authorizations, String tableId)
+ throws ThriftSecurityException, TException {
+
+ Authorizations userauths = null;
+ if (!security.canConditionallyUpdate(credentials, tableId, Tables.getNamespaceId(instance, tableId), authorizations))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ userauths = security.getUserAuthorizations(credentials);
+ for (ByteBuffer auth : authorizations)
+ if (!userauths.contains(ByteBufferUtil.toBytes(auth)))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
+
+ ConditionalSession cs = new ConditionalSession();
+ cs.auths = new Authorizations(authorizations);
+ cs.credentials = credentials;
+ cs.tableId = tableId;
+ cs.interruptFlag = new AtomicBoolean();
+
+ long sid = sessionManager.createSession(cs, false);
+ return new TConditionalSession(sid, lockID, sessionManager.getMaxIdleTime());
+ }
+
+ @Override
+ public List<TCMResult> conditionalUpdate(TInfo tinfo, long sessID, Map<TKeyExtent,List<TConditionalMutation>> mutations, List<String> symbols)
+ throws NoSuchScanIDException, TException {
+
+ ConditionalSession cs = (ConditionalSession) sessionManager.reserveSession(sessID);
+
+ if (cs == null || cs.interruptFlag.get())
+ throw new NoSuchScanIDException();
+
+ if (!cs.tableId.equals(MetadataTable.ID) && !cs.tableId.equals(RootTable.ID))
+ TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
+
+ Text tid = new Text(cs.tableId);
+ long opid = writeTracker.startWrite(TabletType.type(new KeyExtent(tid, null, null)));
+
+ try {
+ Map<KeyExtent,List<ServerConditionalMutation>> updates = Translator.translate(mutations, Translators.TKET,
+ new Translator.ListTranslator<TConditionalMutation,ServerConditionalMutation>(ServerConditionalMutation.TCMT));
+
+ for (KeyExtent ke : updates.keySet())
+ if (!ke.getTableId().equals(tid))
+ throw new IllegalArgumentException("Unexpected table id " + tid + " != " + ke.getTableId());
+
+ ArrayList<TCMResult> results = new ArrayList<TCMResult>();
+
+ Map<KeyExtent,List<ServerConditionalMutation>> deferred = conditionalUpdate(cs, updates, results, symbols);
+
+ while (deferred.size() > 0) {
+ deferred = conditionalUpdate(cs, deferred, results, symbols);
+ }
+
+ return results;
+ } catch (IOException ioe) {
+ throw new TException(ioe);
+ } finally {
+ writeTracker.finishWrite(opid);
+ sessionManager.unreserveSession(sessID);
+ }
+ }
+
+ @Override
+ public void invalidateConditionalUpdate(TInfo tinfo, long sessID) throws TException {
+ // this method should wait for any running conditional update to complete
+ // after this method returns a conditional update should not be able to start
+
+ ConditionalSession cs = (ConditionalSession) sessionManager.getSession(sessID);
+ if (cs != null)
+ cs.interruptFlag.set(true);
+
+ cs = (ConditionalSession) sessionManager.reserveSession(sessID, true);
+ if (cs != null)
+ sessionManager.removeSession(sessID, true);
+ }
+
+ @Override
+ public void closeConditionalUpdate(TInfo tinfo, long sessID) throws TException {
+ sessionManager.removeSession(sessID, false);
+ }
+
+ @Override
+ public void splitTablet(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, ByteBuffer splitPoint) throws NotServingTabletException,
+ ThriftSecurityException {
+
+ String tableId = new String(ByteBufferUtil.toBytes(tkeyExtent.table));
+ String namespaceId;
+ try {
+ namespaceId = Tables.getNamespaceId(instance, tableId);
+ } catch (IllegalArgumentException ex) {
+ // table does not exist, try to educate the client
+ throw new NotServingTabletException(tkeyExtent);
+ }
-
++
+ if (!security.canSplitTablet(credentials, tableId, namespaceId))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ KeyExtent keyExtent = new KeyExtent(tkeyExtent);
+
+ Tablet tablet = onlineTablets.get(keyExtent);
+ if (tablet == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+
+ if (keyExtent.getEndRow() == null || !keyExtent.getEndRow().equals(ByteBufferUtil.toText(splitPoint))) {
+ try {
+ if (TabletServer.this.splitTablet(tablet, ByteBufferUtil.toBytes(splitPoint)) == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+ } catch (IOException e) {
+ log.warn("Failed to split " + keyExtent, e);
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Override
+ public TabletServerStatus getTabletServerStatus(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ return getStats(sessionManager.getActiveScansPerTable());
+ }
+
+ @Override
+ public List<TabletStats> getTabletStats(TInfo tinfo, TCredentials credentials, String tableId) throws ThriftSecurityException, TException {
+ TreeMap<KeyExtent,Tablet> onlineTabletsCopy;
+ synchronized (onlineTablets) {
+ onlineTabletsCopy = new TreeMap<KeyExtent,Tablet>(onlineTablets);
+ }
+ List<TabletStats> result = new ArrayList<TabletStats>();
+ Text text = new Text(tableId);
+ KeyExtent start = new KeyExtent(text, new Text(), null);
+ for (Entry<KeyExtent,Tablet> entry : onlineTabletsCopy.tailMap(start).entrySet()) {
+ KeyExtent ke = entry.getKey();
+ if (ke.getTableId().compareTo(text) == 0) {
+ Tablet tablet = entry.getValue();
+ TabletStats stats = tablet.timer.getTabletStats();
+ stats.extent = ke.toThrift();
+ stats.ingestRate = tablet.ingestRate();
+ stats.queryRate = tablet.queryRate();
+ stats.splitCreationTime = tablet.getSplitCreationTime();
+ stats.numEntries = tablet.getNumEntries();
+ result.add(stats);
+ }
+ }
+ return result;
+ }
+
+ private ZooCache masterLockCache = new ZooCache();
+
+ private void checkPermission(TCredentials credentials, String lock, final String request) throws ThriftSecurityException {
+ boolean fatal = false;
+ try {
+ log.debug("Got " + request + " message from user: " + credentials.getPrincipal());
+ if (!security.canPerformSystemActions(credentials)) {
+ log.warn("Got " + request + " message from user: " + credentials.getPrincipal());
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ }
+ } catch (ThriftSecurityException e) {
+ log.warn("Got " + request + " message from unauthenticatable user: " + e.getUser());
+ if (SystemCredentials.get().getToken().getClass().getName().equals(credentials.getTokenClassName())) {
+ log.fatal("Got message from a service with a mismatched configuration. Please ensure a compatible configuration.", e);
+ fatal = true;
+ }
+ throw e;
+ } finally {
+ if (fatal) {
+ Halt.halt(1, new Runnable() {
+ @Override
+ public void run() {
+ logGCInfo(getSystemConfiguration());
+ }
+ });
+ }
+ }
+
+ if (tabletServerLock == null || !tabletServerLock.wasLockAcquired()) {
+ log.warn("Got " + request + " message from master before lock acquired, ignoring...");
+ throw new RuntimeException("Lock not acquired");
+ }
+
+ if (tabletServerLock != null && tabletServerLock.wasLockAcquired() && !tabletServerLock.isLocked()) {
+ Halt.halt(1, new Runnable() {
+ @Override
+ public void run() {
+ log.info("Tablet server no longer holds lock during checkPermission(
<TRUNCATED>
[40/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --cc minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
index a366c16,0000000..e2b2f83
mode 100644,000000..100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
@@@ -1,392 -1,0 +1,382 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.minicluster;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.gc.SimpleGarbageCollector;
+import org.apache.accumulo.server.master.Master;
+import org.apache.accumulo.server.tabletserver.TabletServer;
+import org.apache.accumulo.server.util.Initialize;
+import org.apache.accumulo.server.util.PortUtils;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.start.Main;
+import org.apache.zookeeper.server.ZooKeeperServerMain;
+
+/**
+ * A utility class that will create Zookeeper and Accumulo processes that write all of their data to a single local directory. This class makes it easy to test
+ * code against a real Accumulo instance. Its much more accurate for testing than MockAccumulo, but much slower than MockAccumulo.
+ *
+ * @since 1.5.0
+ */
+public class MiniAccumuloCluster {
+
+ private static final String INSTANCE_SECRET = "DONTTELL";
+ private static final String INSTANCE_NAME = "miniInstance";
+
+ private static class LogWriter extends Thread {
+ private BufferedReader in;
+ private BufferedWriter out;
+
- /**
- * @throws IOException
- */
+ public LogWriter(InputStream stream, File logFile) throws IOException {
+ this.setDaemon(true);
+ this.in = new BufferedReader(new InputStreamReader(stream, Constants.UTF8));
+ out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(logFile), Constants.UTF8));
+
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ flush();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }, 1000, 1000);
+ }
+
+ public synchronized void flush() throws IOException {
+ if (out != null)
+ out.flush();
+ }
+
+ @Override
+ public void run() {
+ String line;
+
+ try {
+ while ((line = in.readLine()) != null) {
+ out.append(line);
+ out.append("\n");
+ }
+
+ synchronized (this) {
+ out.close();
+ out = null;
+ in.close();
+ }
+
+ } catch (IOException e) {}
+ }
+ }
+
+ private File libDir;
+ private File libExtDir;
+ private File confDir;
+ private File zooKeeperDir;
+ private File accumuloDir;
+ private File zooCfgFile;
+ private File logDir;
+ private File walogDir;
+
+ private Process zooKeeperProcess;
+ private Process masterProcess;
+ private Process gcProcess;
+
+ private int zooKeeperPort;
+
+ private List<LogWriter> logWriters = new ArrayList<MiniAccumuloCluster.LogWriter>();
+
+ private MiniAccumuloConfig config;
+ private Process[] tabletServerProcesses;
+
+ private Process exec(Class<? extends Object> clazz, String... args) throws IOException {
+ String javaHome = System.getProperty("java.home");
+ String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
+ String classpath = System.getProperty("java.class.path");
+
+ classpath = confDir.getAbsolutePath() + File.pathSeparator + classpath;
+
+ String className = clazz.getCanonicalName();
+
+ ArrayList<String> argList = new ArrayList<String>();
+
+ argList.addAll(Arrays.asList(javaBin, "-cp", classpath, "-Xmx128m", "-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75",
+ "-Dapple.awt.UIElement=true", Main.class.getName(), className));
+
+ argList.addAll(Arrays.asList(args));
+
+ ProcessBuilder builder = new ProcessBuilder(argList);
+
+ builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
+ builder.environment().put("ACCUMULO_LOG_DIR", logDir.getAbsolutePath());
+
+ // if we're running under accumulo.start, we forward these env vars
+ String env = System.getenv("HADOOP_PREFIX");
+ if (env != null)
+ builder.environment().put("HADOOP_PREFIX", env);
+ env = System.getenv("ZOOKEEPER_HOME");
+ if (env != null)
+ builder.environment().put("ZOOKEEPER_HOME", env);
+
+ Process process = builder.start();
+
+ LogWriter lw;
+ lw = new LogWriter(process.getErrorStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".err"));
+ logWriters.add(lw);
+ lw.start();
+ lw = new LogWriter(process.getInputStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".out"));
+ logWriters.add(lw);
+ lw.start();
+
+ return process;
+ }
+
+ private void appendProp(Writer fileWriter, Property key, String value, Map<String,String> siteConfig) throws IOException {
+ appendProp(fileWriter, key.getKey(), value, siteConfig);
+ }
+
+ private void appendProp(Writer fileWriter, String key, String value, Map<String,String> siteConfig) throws IOException {
+ if (!siteConfig.containsKey(key))
+ fileWriter.append("<property><name>" + key + "</name><value>" + value + "</value></property>\n");
+ }
+
+ /**
+ * Sets a given key with a random port for the value on the site config if it doesn't already exist.
+ */
+ private void mergePropWithRandomPort(Map<String,String> siteConfig, String key) {
+ if (!siteConfig.containsKey(key)) {
+ siteConfig.put(key, "0");
+ }
+ }
+
+ /**
+ *
+ * @param dir
+ * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
+ * and Junit provide methods for creating temporary directories.
+ * @param rootPassword
+ * Initial root password for instance.
- * @throws IOException
+ */
+ public MiniAccumuloCluster(File dir, String rootPassword) throws IOException {
+ this(new MiniAccumuloConfig(dir, rootPassword));
+ }
+
+ /**
+ * @param config
+ * initial configuration
- * @throws IOException
+ */
+
+ public MiniAccumuloCluster(MiniAccumuloConfig config) throws IOException {
+
+ if (config.getDir().exists() && !config.getDir().isDirectory())
+ throw new IllegalArgumentException("Must pass in directory, " + config.getDir() + " is a file");
+
+ if (config.getDir().exists() && config.getDir().list().length != 0)
+ throw new IllegalArgumentException("Directory " + config.getDir() + " is not empty");
+
+ this.config = config;
+
+ libDir = new File(config.getDir(), "lib");
+ libExtDir = new File(libDir, "ext");
+ confDir = new File(config.getDir(), "conf");
+ accumuloDir = new File(config.getDir(), "accumulo");
+ zooKeeperDir = new File(config.getDir(), "zookeeper");
+ logDir = new File(config.getDir(), "logs");
+ walogDir = new File(config.getDir(), "walogs");
+
+ confDir.mkdirs();
+ accumuloDir.mkdirs();
+ zooKeeperDir.mkdirs();
+ logDir.mkdirs();
+ walogDir.mkdirs();
+ libDir.mkdirs();
+
+ // Avoid the classloader yelling that the general.dynamic.classpaths value is invalid because
+ // $ACCUMULO_HOME/lib/ext isn't defined.
+ libExtDir.mkdirs();
+
+ zooKeeperPort = PortUtils.getRandomFreePort();
+
+ File siteFile = new File(confDir, "accumulo-site.xml");
+
+ OutputStreamWriter fileWriter = new OutputStreamWriter(new FileOutputStream(siteFile), Constants.UTF8);
+ fileWriter.append("<configuration>\n");
+
+ HashMap<String,String> siteConfig = new HashMap<String,String>(config.getSiteConfig());
+
+ appendProp(fileWriter, Property.INSTANCE_DFS_URI, "file:///", siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_DFS_DIR, accumuloDir.getAbsolutePath(), siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_ZK_HOST, "localhost:" + zooKeeperPort, siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_SECRET, INSTANCE_SECRET, siteConfig);
+ appendProp(fileWriter, Property.TSERV_PORTSEARCH, "true", siteConfig);
+ appendProp(fileWriter, Property.LOGGER_DIR, walogDir.getAbsolutePath(), siteConfig);
+ appendProp(fileWriter, Property.TSERV_DATACACHE_SIZE, "10M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_INDEXCACHE_SIZE, "10M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_MAXMEM, "50M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_WALOG_MAX_SIZE, "100M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_NATIVEMAP_ENABLED, "false", siteConfig);
+ appendProp(fileWriter, Property.TRACE_TOKEN_PROPERTY_PREFIX + ".password", config.getRootPassword(), siteConfig);
+ appendProp(fileWriter, Property.GC_CYCLE_DELAY, "4s", siteConfig);
+ appendProp(fileWriter, Property.GC_CYCLE_START, "0s", siteConfig);
+ mergePropWithRandomPort(siteConfig, Property.MASTER_CLIENTPORT.getKey());
+ mergePropWithRandomPort(siteConfig, Property.TRACE_PORT.getKey());
+ mergePropWithRandomPort(siteConfig, Property.TSERV_CLIENTPORT.getKey());
+ mergePropWithRandomPort(siteConfig, Property.MONITOR_PORT.getKey());
+ mergePropWithRandomPort(siteConfig, Property.GC_PORT.getKey());
+
+ // since there is a small amount of memory, check more frequently for majc... setting may not be needed in 1.5
+ appendProp(fileWriter, Property.TSERV_MAJC_DELAY, "3", siteConfig);
+
+ // ACCUMULO-1472 -- Use the classpath, not what might be installed on the system.
+ // We have to set *something* here, otherwise the AccumuloClassLoader will default to pulling from
+ // environment variables (e.g. ACCUMULO_HOME, HADOOP_HOME/PREFIX) which will result in multiple copies
+ // of artifacts on the classpath as they'll be provided by the invoking application
+ appendProp(fileWriter, Property.GENERAL_CLASSPATHS, libDir.getAbsolutePath() + "/[^.].*.jar", siteConfig);
+ appendProp(fileWriter, Property.GENERAL_DYNAMIC_CLASSPATHS, libExtDir.getAbsolutePath() + "/[^.].*.jar", siteConfig);
+
+ for (Entry<String,String> entry : siteConfig.entrySet())
+ fileWriter.append("<property><name>" + entry.getKey() + "</name><value>" + entry.getValue() + "</value></property>\n");
+ fileWriter.append("</configuration>\n");
+ fileWriter.close();
+
+ zooCfgFile = new File(confDir, "zoo.cfg");
+ fileWriter = new OutputStreamWriter(new FileOutputStream(zooCfgFile), Constants.UTF8);
+
+ // zookeeper uses Properties to read its config, so use that to write in order to properly escape things like Windows paths
+ Properties zooCfg = new Properties();
+ zooCfg.setProperty("tickTime", "1000");
+ zooCfg.setProperty("initLimit", "10");
+ zooCfg.setProperty("syncLimit", "5");
+ zooCfg.setProperty("clientPort", zooKeeperPort + "");
+ zooCfg.setProperty("maxClientCnxns", "100");
+ zooCfg.setProperty("dataDir", zooKeeperDir.getAbsolutePath());
+ zooCfg.store(fileWriter, null);
+
+ fileWriter.close();
+
+ }
+
+ /**
+ * Starts Accumulo and Zookeeper processes. Can only be called once.
+ *
- * @throws IOException
- * @throws InterruptedException
+ * @throws IllegalStateException
+ * if already started
+ */
+
+ public void start() throws IOException, InterruptedException {
+ if (zooKeeperProcess != null)
+ throw new IllegalStateException("Already started");
+
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ try {
+ MiniAccumuloCluster.this.stop();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+
+ zooKeeperProcess = exec(Main.class, ZooKeeperServerMain.class.getName(), zooCfgFile.getAbsolutePath());
+
+ // sleep a little bit to let zookeeper come up before calling init, seems to work better
+ UtilWaitThread.sleep(250);
+
+ Process initProcess = exec(Initialize.class, "--instance-name", INSTANCE_NAME, "--password", config.getRootPassword());
+ int ret = initProcess.waitFor();
+ if (ret != 0) {
+ throw new RuntimeException("Initialize process returned " + ret + ". Check the logs in " + logDir + " for errors.");
+ }
+
+ tabletServerProcesses = new Process[config.getNumTservers()];
+ for (int i = 0; i < config.getNumTservers(); i++) {
+ tabletServerProcesses[i] = exec(TabletServer.class);
+ }
+
+ masterProcess = exec(Master.class);
+
+ gcProcess = exec(SimpleGarbageCollector.class);
+ }
+
+ /**
+ * @return Accumulo instance name
+ */
+
+ public String getInstanceName() {
+ return INSTANCE_NAME;
+ }
+
+ /**
+ * @return zookeeper connection string
+ */
+
+ public String getZooKeepers() {
+ return "localhost:" + zooKeeperPort;
+ }
+
+ /**
+ * Stops Accumulo and Zookeeper processes. If stop is not called, there is a shutdown hook that is setup to kill the processes. Howerver its probably best to
+ * call stop in a finally block as soon as possible.
- *
- * @throws IOException
- * @throws InterruptedException
+ */
+
+ public void stop() throws IOException, InterruptedException {
+ if (zooKeeperProcess != null) {
+ zooKeeperProcess.destroy();
+ zooKeeperProcess.waitFor();
+ }
+ if (masterProcess != null) {
+ masterProcess.destroy();
+ masterProcess.waitFor();
+ }
+ if (tabletServerProcesses != null) {
+ for (Process tserver : tabletServerProcesses) {
+ tserver.destroy();
+ tserver.waitFor();
+ }
+ }
+
+ for (LogWriter lw : logWriters)
+ lw.flush();
+
+ if (gcProcess != null) {
+ gcProcess.destroy();
+ gcProcess.waitFor();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/proxy/src/test/java/org/apache/accumulo/proxy/TestProxyReadWrite.java
----------------------------------------------------------------------
diff --cc proxy/src/test/java/org/apache/accumulo/proxy/TestProxyReadWrite.java
index 99a3218,0000000..c0049a0
mode 100644,000000..100644
--- a/proxy/src/test/java/org/apache/accumulo/proxy/TestProxyReadWrite.java
+++ b/proxy/src/test/java/org/apache/accumulo/proxy/TestProxyReadWrite.java
@@@ -1,488 -1,0 +1,478 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.proxy;
+
+import static org.junit.Assert.assertEquals;
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.proxy.thrift.BatchScanOptions;
+import org.apache.accumulo.proxy.thrift.ColumnUpdate;
+import org.apache.accumulo.proxy.thrift.IteratorSetting;
+import org.apache.accumulo.proxy.thrift.Key;
+import org.apache.accumulo.proxy.thrift.KeyValue;
+import org.apache.accumulo.proxy.thrift.Range;
+import org.apache.accumulo.proxy.thrift.ScanColumn;
+import org.apache.accumulo.proxy.thrift.ScanOptions;
+import org.apache.accumulo.proxy.thrift.ScanResult;
+import org.apache.accumulo.proxy.thrift.TimeType;
+import org.apache.thrift.protocol.TCompactProtocol;
+import org.apache.thrift.server.TServer;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestProxyReadWrite {
+ protected static TServer proxy;
+ protected static Thread thread;
+ protected static TestProxyClient tpc;
+ protected static ByteBuffer userpass;
+ protected static final int port = 10194;
+ protected static final String testtable = "testtable";
+
+ @SuppressWarnings("serial")
+ @BeforeClass
+ public static void setup() throws Exception {
+ Properties prop = new Properties();
+ prop.setProperty("useMockInstance", "true");
+ prop.put("tokenClass", PasswordToken.class.getName());
+
+ proxy = Proxy.createProxyServer(Class.forName("org.apache.accumulo.proxy.thrift.AccumuloProxy"), Class.forName("org.apache.accumulo.proxy.ProxyServer"),
+ port, TCompactProtocol.Factory.class, prop);
+ thread = new Thread() {
+ @Override
+ public void run() {
+ proxy.serve();
+ }
+ };
+ thread.start();
+ tpc = new TestProxyClient("localhost", port);
+ userpass = tpc.proxy().login("root", new TreeMap<String, String>() {{put("password",""); }});
+ }
+
+ @AfterClass
+ public static void tearDown() throws InterruptedException {
+ proxy.stop();
+ thread.join();
+ }
+
+ @Before
+ public void makeTestTable() throws Exception {
+ tpc.proxy().createTable(userpass, testtable, true, TimeType.MILLIS);
+ }
+
+ @After
+ public void deleteTestTable() throws Exception {
+ tpc.proxy().deleteTable(userpass, testtable);
+ }
+
+ private static void addMutation(Map<ByteBuffer,List<ColumnUpdate>> mutations, String row, String cf, String cq, String value) {
+ ColumnUpdate update = new ColumnUpdate(ByteBuffer.wrap(cf.getBytes()), ByteBuffer.wrap(cq.getBytes()));
+ update.setValue(value.getBytes());
+ mutations.put(ByteBuffer.wrap(row.getBytes()), Collections.singletonList(update));
+ }
+
+ private static void addMutation(Map<ByteBuffer,List<ColumnUpdate>> mutations, String row, String cf, String cq, String vis, String value) {
+ ColumnUpdate update = new ColumnUpdate(ByteBuffer.wrap(cf.getBytes()), ByteBuffer.wrap(cq.getBytes()));
+ update.setValue(value.getBytes());
+ update.setColVisibility(vis.getBytes());
+ mutations.put(ByteBuffer.wrap(row.getBytes()), Collections.singletonList(update));
+ }
+
+ /**
+ * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Set a range so only the entries between -Inf...5 come back (there should be
+ * 50,000)
- *
- * @throws Exception
+ */
+ @Test
+ public void readWriteBatchOneShotWithRange() throws Exception {
+ int maxInserts = 100000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ for (int i = 0; i < maxInserts; i++) {
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+ tpc.proxy().updateAndFlush(userpass, testtable, mutations);
+ mutations.clear();
+ }
+ }
+
+ Key stop = new Key();
+ stop.setRow("5".getBytes());
+ BatchScanOptions options = new BatchScanOptions();
+ options.ranges = Collections.singletonList(new Range(null, false, stop, false));
+ String cookie = tpc.proxy().createBatchScanner(userpass, testtable, options);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ i += kvList.getResultsSize();
+ hasNext = kvList.isMore();
+ }
+ assertEquals(i, 50000);
+ }
+
+ /**
+ * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Set a columnFamily so only the entries with specified column family come back (there should be
+ * 50,000)
- *
- * @throws Exception
+ */
+ @Test
+ public void readWriteBatchOneShotWithColumnFamilyOnly() throws Exception {
+ int maxInserts = 100000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ for (int i = 0; i < maxInserts; i++) {
+
+ addMutation(mutations, String.format(format, i), "cf" + (i % 2) , "cq" + (i % 2), Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+ tpc.proxy().updateAndFlush(userpass, testtable, mutations);
+ mutations.clear();
+ }
+ }
+
+ BatchScanOptions options = new BatchScanOptions();
+
+ ScanColumn sc = new ScanColumn();
+ sc.colFamily = ByteBuffer.wrap("cf0".getBytes());
+
+ options.columns = Collections.singletonList(sc);
+ String cookie = tpc.proxy().createBatchScanner(userpass, testtable, options);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ i += kvList.getResultsSize();
+ hasNext = kvList.isMore();
+ }
+ assertEquals(i, 50000);
+ }
+
+
+ /**
+ * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Set a columnFamily + columnQualififer so only the entries with specified column
+ * come back (there should be 50,000)
- *
- * @throws Exception
+ */
+ @Test
+ public void readWriteBatchOneShotWithFullColumn() throws Exception {
+ int maxInserts = 100000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ for (int i = 0; i < maxInserts; i++) {
+
+ addMutation(mutations, String.format(format, i), "cf" + (i % 2) , "cq" + (i % 2), Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+ tpc.proxy().updateAndFlush(userpass, testtable, mutations);
+ mutations.clear();
+ }
+ }
+
+ BatchScanOptions options = new BatchScanOptions();
+
+ ScanColumn sc = new ScanColumn();
+ sc.colFamily = ByteBuffer.wrap("cf0".getBytes());
+ sc.colQualifier = ByteBuffer.wrap("cq0".getBytes());
+
+ options.columns = Collections.singletonList(sc);
+ String cookie = tpc.proxy().createBatchScanner(userpass, testtable, options);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ i += kvList.getResultsSize();
+ hasNext = kvList.isMore();
+ }
+ assertEquals(i, 50000);
+ }
+
+
+ /**
+ * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Filter the results so only the even numbers come back.
- *
- * @throws Exception
+ */
+ @Test
+ public void readWriteBatchOneShotWithFilterIterator() throws Exception {
+ int maxInserts = 10000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ for (int i = 0; i < maxInserts; i++) {
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+ tpc.proxy().updateAndFlush(userpass, testtable, mutations);
+ mutations.clear();
+ }
+
+ }
+
+ String regex = ".*[02468]";
+
+ org.apache.accumulo.core.client.IteratorSetting is = new org.apache.accumulo.core.client.IteratorSetting(50, regex, RegExFilter.class);
+ RegExFilter.setRegexs(is, regex, null, null, null, false);
+
+ IteratorSetting pis = Util.iteratorSetting2ProxyIteratorSetting(is);
+ ScanOptions opts = new ScanOptions();
+ opts.iterators = Collections.singletonList(pis);
+ String cookie = tpc.proxy().createScanner(userpass, testtable, opts);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ for (KeyValue kv : kvList.getResults()) {
+ assertEquals(Integer.parseInt(new String(kv.getKey().getRow())), i);
+
+ i += 2;
+ }
+ hasNext = kvList.isMore();
+ }
+ }
+
+ @Test
+ public void readWriteOneShotWithRange() throws Exception {
+ int maxInserts = 100000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ for (int i = 0; i < maxInserts; i++) {
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+ tpc.proxy().updateAndFlush(userpass, testtable, mutations);
+ mutations.clear();
+ }
+ }
+
+ Key stop = new Key();
+ stop.setRow("5".getBytes());
+ ScanOptions opts = new ScanOptions();
+ opts.range = new Range(null, false, stop, false);
+ String cookie = tpc.proxy().createScanner(userpass, testtable, opts);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ i += kvList.getResultsSize();
+ hasNext = kvList.isMore();
+ }
+ assertEquals(i, 50000);
+ }
+
+ /**
+ * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Filter the results so only the even numbers come back.
- *
- * @throws Exception
+ */
+ @Test
+ public void readWriteOneShotWithFilterIterator() throws Exception {
+ int maxInserts = 10000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ for (int i = 0; i < maxInserts; i++) {
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+
+ tpc.proxy().updateAndFlush(userpass, testtable, mutations);
+ mutations.clear();
+
+ }
+
+ }
+
+ String regex = ".*[02468]";
+
+ org.apache.accumulo.core.client.IteratorSetting is = new org.apache.accumulo.core.client.IteratorSetting(50, regex, RegExFilter.class);
+ RegExFilter.setRegexs(is, regex, null, null, null, false);
+
+ IteratorSetting pis = Util.iteratorSetting2ProxyIteratorSetting(is);
+ ScanOptions opts = new ScanOptions();
+ opts.iterators = Collections.singletonList(pis);
+ String cookie = tpc.proxy().createScanner(userpass, testtable, opts);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ for (KeyValue kv : kvList.getResults()) {
+ assertEquals(Integer.parseInt(new String(kv.getKey().getRow())), i);
+
+ i += 2;
+ }
+ hasNext = kvList.isMore();
+ }
+ }
+
+ // @Test
+ // This test takes kind of a long time. Enable it if you think you may have memory issues.
+ public void manyWritesAndReads() throws Exception {
+ int maxInserts = 1000000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$06d";
+ String writer = tpc.proxy().createWriter(userpass, testtable, null);
+ for (int i = 0; i < maxInserts; i++) {
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+
+ tpc.proxy().update(writer, mutations);
+ mutations.clear();
+
+ }
+
+ }
+
+ tpc.proxy().flush(writer);
+ tpc.proxy().closeWriter(writer);
+
+ String cookie = tpc.proxy().createScanner(userpass, testtable, null);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ for (KeyValue kv : kvList.getResults()) {
+ assertEquals(Integer.parseInt(new String(kv.getKey().getRow())), i);
+ i++;
+ }
+ hasNext = kvList.isMore();
+ if (hasNext)
+ assertEquals(k, kvList.getResults().size());
+ }
+ assertEquals(maxInserts, i);
+ }
+
+ @Test
+ public void asynchReadWrite() throws Exception {
+ int maxInserts = 10000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ String writer = tpc.proxy().createWriter(userpass, testtable, null);
+ for (int i = 0; i < maxInserts; i++) {
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+ tpc.proxy().update(writer, mutations);
+ mutations.clear();
+ }
+ }
+
+ tpc.proxy().flush(writer);
+ tpc.proxy().closeWriter(writer);
+
+ String regex = ".*[02468]";
+
+ org.apache.accumulo.core.client.IteratorSetting is = new org.apache.accumulo.core.client.IteratorSetting(50, regex, RegExFilter.class);
+ RegExFilter.setRegexs(is, regex, null, null, null, false);
+
+ IteratorSetting pis = Util.iteratorSetting2ProxyIteratorSetting(is);
+ ScanOptions opts = new ScanOptions();
+ opts.iterators = Collections.singletonList(pis);
+ String cookie = tpc.proxy().createScanner(userpass, testtable, opts);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ int numRead = 0;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ for (KeyValue kv : kvList.getResults()) {
+ assertEquals(i, Integer.parseInt(new String(kv.getKey().getRow())));
+ numRead++;
+ i += 2;
+ }
+ hasNext = kvList.isMore();
+ }
+ assertEquals(maxInserts / 2, numRead);
+ }
+
+ @Test
+ public void testVisibility() throws Exception {
+
+ Set<ByteBuffer> auths = new HashSet<ByteBuffer>();
+ auths.add(ByteBuffer.wrap("even".getBytes()));
+ tpc.proxy().changeUserAuthorizations(userpass, "root", auths);
+
+ int maxInserts = 10000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ String writer = tpc.proxy().createWriter(userpass, testtable, null);
+ for (int i = 0; i < maxInserts; i++) {
+ if (i % 2 == 0)
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, "even", Util.randString(10));
+ else
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, "odd", Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+ tpc.proxy().update(writer, mutations);
+ mutations.clear();
+ }
+ }
+
+ tpc.proxy().flush(writer);
+ tpc.proxy().closeWriter(writer);
+ ScanOptions opts = new ScanOptions();
+ opts.authorizations = auths;
+ String cookie = tpc.proxy().createScanner(userpass, testtable, opts);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ int numRead = 0;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ for (KeyValue kv : kvList.getResults()) {
+ assertEquals(Integer.parseInt(new String(kv.getKey().getRow())), i);
+ i += 2;
+ numRead++;
+ }
+ hasNext = kvList.isMore();
+
+ }
+ assertEquals(maxInserts / 2, numRead);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java
index 442294f,0000000..05806ca
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java
+++ b/server/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java
@@@ -1,30 -1,0 +1,27 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.conf;
+
+import org.apache.accumulo.server.client.HdfsZooInstance;
+
+public class ConfigSanityCheck {
+
- /**
- * @param args
- */
+ public static void main(String[] args) {
+ new ServerConfiguration(HdfsZooInstance.getInstance()).getConfiguration();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
index 4f9d33a,0000000..01626ad
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
+++ b/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
@@@ -1,191 -1,0 +1,190 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.logger;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.tabletserver.log.DfsLogger;
+import org.apache.accumulo.server.tabletserver.log.MultiReader;
+import org.apache.accumulo.server.trace.TraceFileSystem;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+
+public class LogReader {
+
+ static class Opts extends Help {
+ @Parameter(names = "-r", description = "print only mutations associated with the given row")
+ String row;
+ @Parameter(names = "-m", description = "limit the number of mutations printed per row")
+ int maxMutations = 5;
+ @Parameter(names = "-t", description = "print only mutations that fall within the given key extent")
+ String extent;
+ @Parameter(names = "-p", description = "search for a row that matches the given regex")
+ String regexp;
+ @Parameter(description = "<logfile> { <logfile> ... }")
+ List<String> files = new ArrayList<String>();
+ }
+
+ /**
+ * Dump a Log File (Map or Sequence) to stdout. Will read from HDFS or local file system.
+ *
+ * @param args
+ * - first argument is the file to print
- * @throws IOException
+ */
+ public static void main(String[] args) throws IOException {
+ Opts opts = new Opts();
+ opts.parseArgs(LogReader.class.getName(), args);
+ Configuration conf = CachedConfiguration.getInstance();
+ FileSystem fs = TraceFileSystem.wrap(FileUtil.getFileSystem(conf, ServerConfiguration.getSiteConfiguration()));
+ FileSystem local = TraceFileSystem.wrap(FileSystem.getLocal(conf));
+
+ Matcher rowMatcher = null;
+ KeyExtent ke = null;
+ Text row = null;
+ if (opts.files.isEmpty()) {
+ new JCommander(opts).usage();
+ return;
+ }
+ if (opts.row != null)
+ row = new Text(opts.row);
+ if (opts.extent != null) {
+ String sa[] = opts.extent.split(";");
+ ke = new KeyExtent(new Text(sa[0]), new Text(sa[1]), new Text(sa[2]));
+ }
+ if (opts.regexp != null) {
+ Pattern pattern = Pattern.compile(opts.regexp);
+ rowMatcher = pattern.matcher("");
+ }
+
+ Set<Integer> tabletIds = new HashSet<Integer>();
+
+ for (String file : opts.files) {
+
+ Map<String, String> meta = new HashMap<String, String>();
+ Path path = new Path(file);
+ LogFileKey key = new LogFileKey();
+ LogFileValue value = new LogFileValue();
+
+ if (fs.isFile(path)) {
+ // read log entries from a simple hdfs file
+ FSDataInputStream f = DfsLogger.readHeader(fs, path, meta);
+ try {
+ while (true) {
+ try {
+ key.readFields(f);
+ value.readFields(f);
+ } catch (EOFException ex) {
+ break;
+ }
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
+ }
+ } finally {
+ f.close();
+ }
+ } else if (local.isFile(path)) {
+ // read log entries from a simple file
+ FSDataInputStream f = DfsLogger.readHeader(fs, path, meta);
+ try {
+ while (true) {
+ try {
+ key.readFields(f);
+ value.readFields(f);
+ } catch (EOFException ex) {
+ break;
+ }
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
+ }
+ } finally {
+ f.close();
+ }
+ } else {
+ // read the log entries sorted in a map file
+ MultiReader input = new MultiReader(fs, conf, file);
+ while (input.next(key, value)) {
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
+ }
+ }
+ }
+ }
+
+ public static void printLogEvent(LogFileKey key, LogFileValue value, Text row, Matcher rowMatcher, KeyExtent ke, Set<Integer> tabletIds, int maxMutations) {
+
+ if (ke != null) {
+ if (key.event == LogEvents.DEFINE_TABLET) {
+ if (key.tablet.equals(ke)) {
+ tabletIds.add(key.tid);
+ } else {
+ return;
+ }
+ } else if (!tabletIds.contains(key.tid)) {
+ return;
+ }
+ }
+
+ if (row != null || rowMatcher != null) {
+ if (key.event == LogEvents.MUTATION || key.event == LogEvents.MANY_MUTATIONS) {
+ boolean found = false;
+ for (Mutation m : value.mutations) {
+ if (row != null && new Text(m.getRow()).equals(row)) {
+ found = true;
+ break;
+ }
+
+ if (rowMatcher != null) {
+ rowMatcher.reset(new String(m.getRow(), Constants.UTF8));
+ if (rowMatcher.matches()) {
+ found = true;
+ break;
+ }
+ }
+ }
+
+ if (!found)
+ return;
+ } else {
+ return;
+ }
+
+ }
+
+ System.out.println(key);
+ System.out.println(LogFileValue.format(value, maxMutations));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java
index 4930bc2,0000000..e14008a
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java
@@@ -1,152 -1,0 +1,144 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.balancer;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+import java.util.SortedMap;
+
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.master.thrift.TableInfo;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletMigration;
+import org.apache.thrift.TException;
+
+/**
+ * A chaotic load balancer used for testing. It constantly shuffles tablets, preventing them from resting in a single location for very long. This is not
+ * designed for performance, do not use on production systems. I'm calling it the LokiLoadBalancer.
+ */
+public class ChaoticLoadBalancer extends TabletBalancer {
+ Random r = new Random();
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.master.balancer.TabletBalancer#getAssignments(java.util.SortedMap, java.util.Map, java.util.Map)
- */
+ @Override
+ public void getAssignments(SortedMap<TServerInstance,TabletServerStatus> current, Map<KeyExtent,TServerInstance> unassigned,
+ Map<KeyExtent,TServerInstance> assignments) {
+ long total = assignments.size() + unassigned.size();
+ long avg = (long) Math.ceil(((double) total) / current.size());
+ Map<TServerInstance,Long> toAssign = new HashMap<TServerInstance,Long>();
+ List<TServerInstance> tServerArray = new ArrayList<TServerInstance>();
+ for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
+ long numTablets = 0;
+ for (TableInfo ti : e.getValue().getTableMap().values()) {
+ numTablets += ti.tablets;
+ }
+ if (numTablets < avg) {
+ tServerArray.add(e.getKey());
+ toAssign.put(e.getKey(), avg - numTablets);
+ }
+ }
+
+ for (KeyExtent ke : unassigned.keySet())
+ {
+ int index = r.nextInt(tServerArray.size());
+ TServerInstance dest = tServerArray.get(index);
+ assignments.put(ke, dest);
+ long remaining = toAssign.get(dest).longValue() - 1;
+ if (remaining == 0) {
+ tServerArray.remove(index);
+ toAssign.remove(dest);
+ } else {
+ toAssign.put(dest, remaining);
+ }
+ }
+ }
+
+ /**
+ * Will balance randomly, maintaining distribution
+ */
+ @Override
+ public long balance(SortedMap<TServerInstance,TabletServerStatus> current, Set<KeyExtent> migrations, List<TabletMigration> migrationsOut) {
+ Map<TServerInstance,Long> numTablets = new HashMap<TServerInstance,Long>();
+ List<TServerInstance> underCapacityTServer = new ArrayList<TServerInstance>();
+
+ if (!migrations.isEmpty())
+ return 100;
+
+ boolean moveMetadata = r.nextInt(4) == 0;
+ long totalTablets = 0;
+ for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
+ long tabletCount = 0;
+ for (TableInfo ti : e.getValue().getTableMap().values()) {
+ tabletCount += ti.tablets;
+ }
+ numTablets.put(e.getKey(), tabletCount);
+ underCapacityTServer.add(e.getKey());
+ totalTablets += tabletCount;
+ }
+ // totalTablets is fuzzy due to asynchronicity of the stats
+ // *1.2 to handle fuzziness, and prevent locking for 'perfect' balancing scenarios
+ long avg = (long) Math.ceil(((double) totalTablets) / current.size() * 1.2);
+
+ for (Entry<TServerInstance, TabletServerStatus> e : current.entrySet())
+ {
+ for (String table : e.getValue().getTableMap().keySet())
+ {
+ if (!moveMetadata && "!METADATA".equals(table))
+ continue;
+ try {
+ for (TabletStats ts : getOnlineTabletsForTable(e.getKey(), table)) {
+ KeyExtent ke = new KeyExtent(ts.extent);
+ int index = r.nextInt(underCapacityTServer.size());
+ TServerInstance dest = underCapacityTServer.get(index);
+ if (dest.equals(e.getKey()))
+ continue;
+ migrationsOut.add(new TabletMigration(ke, e.getKey(), dest));
+ if (numTablets.put(dest, numTablets.get(dest) + 1) > avg)
+ underCapacityTServer.remove(index);
+ if (numTablets.put(e.getKey(), numTablets.get(e.getKey()) - 1) <= avg && !underCapacityTServer.contains(e.getKey()))
+ underCapacityTServer.add(e.getKey());
+
+ // We can get some craziness with only 1 tserver, so lets make sure there's always an option!
+ if (underCapacityTServer.isEmpty())
+ underCapacityTServer.addAll(numTablets.keySet());
+ }
+ } catch (ThriftSecurityException e1) {
+ // Shouldn't happen, but carry on if it does
+ e1.printStackTrace();
+ } catch (TException e1) {
+ // Shouldn't happen, but carry on if it does
+ e1.printStackTrace();
+ }
+ }
+ }
+
+ return 100;
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.server.master.balancer.TabletBalancer#init(org.apache.accumulo.server.conf.ServerConfiguration)
- */
+ @Override
+ public void init(ServerConfiguration conf) {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
index d6dce2f,0000000..69387d3
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
@@@ -1,150 -1,0 +1,148 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.balancer;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletMigration;
+import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransportException;
+
+public abstract class TabletBalancer {
+
+ private static final Logger log = Logger.getLogger(TabletBalancer.class);
+
+ protected ServerConfiguration configuration;
+
+ /**
+ * Initialize the TabletBalancer. This gives the balancer the opportunity to read the configuration.
+ */
+ public void init(ServerConfiguration conf) {
+ configuration = conf;
+ }
+
+ /**
+ * Assign tablets to tablet servers. This method is called whenever the master finds tablets that are unassigned.
+ *
+ * @param current
+ * The current table-summary state of all the online tablet servers. Read-only. The TabletServerStatus for each server may be null if the tablet
+ * server has not yet responded to a recent request for status.
+ * @param unassigned
+ * A map from unassigned tablet to the last known tablet server. Read-only.
+ * @param assignments
+ * A map from tablet to assigned server. Write-only.
+ */
+ abstract public void getAssignments(SortedMap<TServerInstance,TabletServerStatus> current, Map<KeyExtent,TServerInstance> unassigned,
+ Map<KeyExtent,TServerInstance> assignments);
+
+ /**
+ * Ask the balancer if any migrations are necessary.
+ *
+ * @param current
+ * The current table-summary state of all the online tablet servers. Read-only.
+ * @param migrations
+ * the current set of migrations. Read-only.
+ * @param migrationsOut
+ * new migrations to perform; should not contain tablets in the current set of migrations. Write-only.
+ * @return the time, in milliseconds, to wait before re-balancing.
+ *
+ * This method will not be called when there are unassigned tablets.
+ */
+ public abstract long balance(SortedMap<TServerInstance,TabletServerStatus> current, Set<KeyExtent> migrations, List<TabletMigration> migrationsOut);
+
+ /**
+ * Fetch the tablets for the given table by asking the tablet server. Useful if your balance strategy needs details at the tablet level to decide what tablets
+ * to move.
+ *
+ * @param tserver
+ * The tablet server to ask.
+ * @param tableId
+ * The table id
+ * @return a list of tablet statistics
+ * @throws ThriftSecurityException
+ * tablet server disapproves of your internal System password.
+ * @throws TException
+ * any other problem
+ */
+ public List<TabletStats> getOnlineTabletsForTable(TServerInstance tserver, String tableId) throws ThriftSecurityException, TException {
+ log.debug("Scanning tablet server " + tserver + " for table " + tableId);
+ Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), tserver.getLocation(), configuration.getConfiguration());
+ try {
+ List<TabletStats> onlineTabletsForTable = client.getTabletStats(Tracer.traceInfo(), SecurityConstants.getSystemCredentials(), tableId);
+ return onlineTabletsForTable;
+ } catch (TTransportException e) {
+ log.error("Unable to connect to " + tserver + ": " + e);
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ return null;
+ }
+
+ /**
+ * Utility to ensure that the migrations from balance() are consistent:
+ * <ul>
+ * <li>Tablet objects are not null
+ * <li>Source and destination tablet servers are not null and current
+ * </ul>
+ *
- * @param current
- * @param migrations
+ * @return A list of TabletMigration object that passed sanity checks.
+ */
+ public static List<TabletMigration> checkMigrationSanity(Set<TServerInstance> current, List<TabletMigration> migrations) {
+ List<TabletMigration> result = new ArrayList<TabletMigration>(migrations.size());
+ for (TabletMigration m : migrations) {
+ if (m.tablet == null) {
+ log.warn("Balancer gave back a null tablet " + m);
+ continue;
+ }
+ if (m.newServer == null) {
+ log.warn("Balancer did not set the destination " + m);
+ continue;
+ }
+ if (m.oldServer == null) {
+ log.warn("Balancer did not set the source " + m);
+ continue;
+ }
+ if (!current.contains(m.oldServer)) {
+ log.warn("Balancer wants to move a tablet from a server that is not current: " + m);
+ continue;
+ }
+ if (!current.contains(m.newServer)) {
+ log.warn("Balancer wants to move a tablet to a server that is not current: " + m);
+ continue;
+ }
+ result.add(m);
+ }
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
index f9f03bd,0000000..540ebc0
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
@@@ -1,87 -1,0 +1,81 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.state;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+
+/**
+ * Interface for storing information about tablet assignments. There are three implementations:
+ *
+ * ZooTabletStateStore: information about the root tablet is stored in ZooKeeper MetaDataStateStore: information about the other tablets are stored in the
+ * metadata table
+ *
+ */
+public abstract class TabletStateStore implements Iterable<TabletLocationState> {
+
+ /**
+ * Identifying name for this tablet state store.
+ */
+ abstract public String name();
+
+ /**
+ * Scan the information about the tablets covered by this store
+ */
++ @Override
+ abstract public Iterator<TabletLocationState> iterator();
+
+ /**
+ * Store the assigned locations in the data store.
- *
- * @param assignments
- * @throws DistributedStoreException
+ */
+ abstract public void setFutureLocations(Collection<Assignment> assignments) throws DistributedStoreException;
+
+ /**
+ * Tablet servers will update the data store with the location when they bring the tablet online
- *
- * @param assignments
- * @throws DistributedStoreException
+ */
+ abstract public void setLocations(Collection<Assignment> assignments) throws DistributedStoreException;
+
+ /**
+ * Mark the tablets as having no known or future location.
+ *
+ * @param tablets
+ * the tablets' current information
- * @throws DistributedStoreException
+ */
+ abstract public void unassign(Collection<TabletLocationState> tablets) throws DistributedStoreException;
+
+ public static void unassign(TabletLocationState tls) throws DistributedStoreException {
+ TabletStateStore store;
+ if (tls.extent.isRootTablet()) {
+ store = new ZooTabletStateStore();
+ } else {
+ store = new MetaDataStateStore();
+ }
+ store.unassign(Collections.singletonList(tls));
+ }
+
+ public static void setLocation(Assignment assignment) throws DistributedStoreException {
+ TabletStateStore store;
+ if (assignment.tablet.isRootTablet()) {
+ store = new ZooTabletStateStore();
+ } else {
+ store = new MetaDataStateStore();
+ }
+ store.setLocations(Collections.singletonList(assignment));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/master/tableOps/TraceRepo.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/master/tableOps/TraceRepo.java
index 58a337f,0000000..45f6a60
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/master/tableOps/TraceRepo.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/tableOps/TraceRepo.java
@@@ -1,109 -1,0 +1,84 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.tableOps;
+
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.instrument.Trace;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.accumulo.trace.thrift.TInfo;
+import org.apache.accumulo.fate.Repo;
+
+
+/**
+ *
+ */
+public class TraceRepo<T> implements Repo<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ TInfo tinfo;
+ Repo<T> repo;
+
+ public TraceRepo(Repo<T> repo) {
+ this.repo = repo;
+ tinfo = Tracer.traceInfo();
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.server.fate.Repo#isReady(long, java.lang.Object)
- */
+ @Override
+ public long isReady(long tid, T environment) throws Exception {
+ Span span = Trace.trace(tinfo, repo.getDescription());
+ try {
+ return repo.isReady(tid, environment);
+ } finally {
+ span.stop();
+ }
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.server.fate.Repo#call(long, java.lang.Object)
- */
+ @Override
+ public Repo<T> call(long tid, T environment) throws Exception {
+ Span span = Trace.trace(tinfo, repo.getDescription());
+ try {
+ Repo<T> result = repo.call(tid, environment);
+ if (result == null)
+ return result;
+ return new TraceRepo<T>(result);
+ } finally {
+ span.stop();
+ }
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.server.fate.Repo#undo(long, java.lang.Object)
- */
+ @Override
+ public void undo(long tid, T environment) throws Exception {
+ Span span = Trace.trace(tinfo, repo.getDescription());
+ try {
+ repo.undo(tid, environment);
+ } finally {
+ span.stop();
+ }
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.server.fate.Repo#getDescription()
- */
+ @Override
+ public String getDescription() {
+ return repo.getDescription();
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.server.fate.Repo#getReturn()
- */
+ @Override
+ public String getReturn() {
+ return repo.getReturn();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/metanalysis/LogFileOutputFormat.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/metanalysis/LogFileOutputFormat.java
index 829d7bc,0000000..7e50754
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/metanalysis/LogFileOutputFormat.java
+++ b/server/src/main/java/org/apache/accumulo/server/metanalysis/LogFileOutputFormat.java
@@@ -1,70 -1,0 +1,66 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.metanalysis;
+
+import java.io.IOException;
+
+import org.apache.accumulo.server.logger.LogFileKey;
+import org.apache.accumulo.server.logger.LogFileValue;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+
+/**
+ * Output format for Accumulo write ahead logs.
+ */
+public class LogFileOutputFormat extends FileOutputFormat<LogFileKey,LogFileValue> {
+
+ private static class LogFileRecordWriter extends RecordWriter<LogFileKey,LogFileValue> {
+
+ private FSDataOutputStream out;
+
- /**
- * @param outputPath
- * @throws IOException
- */
+ public LogFileRecordWriter(Path outputPath) throws IOException {
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+
+ out = fs.create(outputPath);
+ }
+
+ @Override
+ public void close(TaskAttemptContext arg0) throws IOException, InterruptedException {
+ out.close();
+ }
+
+ @Override
+ public void write(LogFileKey key, LogFileValue val) throws IOException, InterruptedException {
+ key.write(out);
+ val.write(out);
+ }
+
+ }
+
+ @Override
+ public RecordWriter<LogFileKey,LogFileValue> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
+ Path outputPath = getDefaultWorkFile(context, "");
+ return new LogFileRecordWriter(outputPath);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/metanalysis/PrintEvents.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/metanalysis/PrintEvents.java
index 88f5cbe,0000000..0478d83
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/metanalysis/PrintEvents.java
+++ b/server/src/main/java/org/apache/accumulo/server/metanalysis/PrintEvents.java
@@@ -1,109 -1,0 +1,106 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.metanalysis;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.Constants;
- import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
++import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.server.logger.LogFileValue;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Looks up and prints mutations indexed by IndexMeta
+ */
+public class PrintEvents {
+
+ static class Opts extends ClientOpts {
+ @Parameter(names={"-t", "--tableId"}, description="table id", required=true)
+ String tableId;
+ @Parameter(names={"-e", "--endRow"}, description="end row")
+ String endRow;
+ @Parameter(names={"-t", "--time"}, description="time, in milliseconds", required=true)
+ long time;
+ }
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(PrintEvents.class.getName(), args);
+
+ Connector conn = opts.getConnector();
+
+ printEvents(conn, opts.tableId, opts.endRow, opts.time);
+ }
+
+ /**
+ * @param conn
+ * @param tablePrefix
+ * @param tableId
+ * @param endRow
+ * @param time
+ */
+ private static void printEvents(Connector conn, String tableId, String endRow, Long time) throws Exception {
+ Scanner scanner = conn.createScanner("tabletEvents", new Authorizations());
+ String metaRow = tableId + (endRow == null ? "<" : ";" + endRow);
+ scanner.setRange(new Range(new Key(metaRow, String.format("%020d", time)), true, new Key(metaRow).followingKey(PartialKey.ROW), false));
+ int count = 0;
+
+ String lastLog = null;
+
+ loop1: for (Entry<Key,Value> entry : scanner) {
+ if (entry.getKey().getColumnQualifier().toString().equals("log")) {
+ if (lastLog == null || !lastLog.equals(entry.getValue().toString()))
+ System.out.println("Log : " + entry.getValue());
+ lastLog = entry.getValue().toString();
+ } else if (entry.getKey().getColumnQualifier().toString().equals("mut")) {
+ DataInputStream dis = new DataInputStream(new ByteArrayInputStream(entry.getValue().get()));
+ Mutation m = new Mutation();
+ m.readFields(dis);
+
+ LogFileValue lfv = new LogFileValue();
+ lfv.mutations = Collections.singletonList(m);
+
+ System.out.println(LogFileValue.format(lfv, 1));
+
+ List<ColumnUpdate> columnsUpdates = m.getUpdates();
+ for (ColumnUpdate cu : columnsUpdates) {
+ if (Constants.METADATA_PREV_ROW_COLUMN.equals(new Text(cu.getColumnFamily()), new Text(cu.getColumnQualifier())) && count > 0) {
+ System.out.println("Saw change to prevrow, stopping printing events.");
+ break loop1;
+ }
+ }
+ count++;
+ }
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
index 5a8ddec,0000000..d76c7a3
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
+++ b/server/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
@@@ -1,277 -1,0 +1,273 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.metrics;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.lang.management.ManagementFactory;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+import javax.management.StandardMBean;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.commons.lang.builder.ToStringBuilder;
+import org.apache.commons.lang.time.DateUtils;
+
+public abstract class AbstractMetricsImpl {
+
+ public class Metric {
+
+ private long count = 0;
+ private long avg = 0;
+ private long min = 0;
+ private long max = 0;
+
+ public long getCount() {
+ return count;
+ }
+
+ public long getAvg() {
+ return avg;
+ }
+
+ public long getMin() {
+ return min;
+ }
+
+ public long getMax() {
+ return max;
+ }
+
+ public void incCount() {
+ count++;
+ }
+
+ public void addAvg(long a) {
+ if (a < 0)
+ return;
+ avg = (long) ((avg * .8) + (a * .2));
+ }
+
+ public void addMin(long a) {
+ if (a < 0)
+ return;
+ min = Math.min(min, a);
+ }
+
+ public void addMax(long a) {
+ if (a < 0)
+ return;
+ max = Math.max(max, a);
+ }
+
+ @Override
+ public String toString() {
+ return new ToStringBuilder(this).append("count", count).append("average", avg).append("minimum", min).append("maximum", max).toString();
+ }
+
+ }
+
+ static final org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger(AbstractMetricsImpl.class);
+
+ private static ConcurrentHashMap<String,Metric> registry = new ConcurrentHashMap<String,Metric>();
+
+ private boolean currentlyLogging = false;
+
+ private File logDir = null;
+
+ private String metricsPrefix = null;
+
+ private Date today = new Date();
+
+ private File logFile = null;
+
+ private Writer logWriter = null;
+
+ private SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd");
+
+ private SimpleDateFormat logFormatter = new SimpleDateFormat("yyyyMMddhhmmssz");
+
+ private MetricsConfiguration config = null;
+
+ public AbstractMetricsImpl() {
+ this.metricsPrefix = getMetricsPrefix();
+ config = new MetricsConfiguration(metricsPrefix);
+ }
+
+ /**
+ * Registers a StandardMBean with the MBean Server
- *
- * @throws Exception
+ */
+ public void register(StandardMBean mbean) throws Exception {
+ // Register this object with the MBeanServer
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ if (null == getObjectName())
+ throw new IllegalArgumentException("MBean object name must be set.");
+ mbs.registerMBean(mbean, getObjectName());
+
+ setupLogging();
+ }
+
+ /**
+ * Registers this MBean with the MBean Server
- *
- * @throws Exception
+ */
+ public void register() throws Exception {
+ // Register this object with the MBeanServer
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ if (null == getObjectName())
+ throw new IllegalArgumentException("MBean object name must be set.");
+ mbs.registerMBean(this, getObjectName());
+
+ setupLogging();
+ }
+
+ public void createMetric(String name) {
+ registry.put(name, new Metric());
+ }
+
+ public Metric getMetric(String name) {
+ return registry.get(name);
+ }
+
+ public long getMetricCount(String name) {
+ return registry.get(name).getCount();
+ }
+
+ public long getMetricAvg(String name) {
+ return registry.get(name).getAvg();
+ }
+
+ public long getMetricMin(String name) {
+ return registry.get(name).getMin();
+ }
+
+ public long getMetricMax(String name) {
+ return registry.get(name).getMax();
+ }
+
+ private void setupLogging() throws IOException {
+ if (null == config.getMetricsConfiguration())
+ return;
+ // If we are already logging, then return
+ if (!currentlyLogging && config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ // Check to see if directory exists, else make it
+ String mDir = config.getMetricsConfiguration().getString("logging.dir");
+ if (null != mDir) {
+ File dir = new File(mDir);
+ if (!dir.isDirectory())
+ if (!dir.mkdir())
+ log.warn("Could not create log directory: " + dir);
+ logDir = dir;
+ // Create new log file
+ startNewLog();
+ }
+ currentlyLogging = true;
+ }
+ }
+
+ private void startNewLog() throws IOException {
+ if (null != logWriter) {
+ logWriter.flush();
+ logWriter.close();
+ }
+ logFile = new File(logDir, metricsPrefix + "-" + formatter.format(today) + ".log");
+ if (!logFile.exists()) {
+ if (!logFile.createNewFile()) {
+ log.error("Unable to create new log file");
+ currentlyLogging = false;
+ return;
+ }
+ }
+ logWriter = new OutputStreamWriter(new FileOutputStream(logFile, true), Constants.UTF8);
+ }
+
+ private void writeToLog(String name) throws IOException {
+ if (null == logWriter)
+ return;
+ // Increment the date if we have to
+ Date now = new Date();
+ if (!DateUtils.isSameDay(today, now)) {
+ today = now;
+ startNewLog();
+ }
+ logWriter.append(logFormatter.format(now)).append(" Metric: ").append(name).append(": ").append(registry.get(name).toString()).append("\n");
+ }
+
+ public void add(String name, long time) {
+ if (isEnabled()) {
+ registry.get(name).incCount();
+ registry.get(name).addAvg(time);
+ registry.get(name).addMin(time);
+ registry.get(name).addMax(time);
+ // If we are not currently logging and should be, then initialize
+ if (!currentlyLogging && config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ try {
+ setupLogging();
+ } catch (IOException ioe) {
+ log.error("Error setting up log", ioe);
+ }
+ } else if (currentlyLogging && !config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ // if we are currently logging and shouldn't be, then close logs
+ try {
+ logWriter.flush();
+ logWriter.close();
+ logWriter = null;
+ logFile = null;
+ } catch (Exception e) {
+ log.error("Error stopping metrics logging", e);
+ }
+ currentlyLogging = false;
+ }
+ if (currentlyLogging) {
+ try {
+ writeToLog(name);
+ } catch (IOException ioe) {
+ log.error("Error writing to metrics log", ioe);
+ }
+ }
+ }
+ }
+
+ public boolean isEnabled() {
+ return config.isEnabled();
+ }
+
+ protected abstract ObjectName getObjectName();
+
+ protected abstract String getMetricsPrefix();
+
+ @Override
+ protected void finalize() {
+ if (null != logWriter) {
+ try {
+ logWriter.close();
+ } catch (Exception e) {
+ // do nothing
+ } finally {
+ logWriter = null;
+ }
+ }
+ logFile = null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/security/handler/InsecurePermHandler.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/security/handler/InsecurePermHandler.java
index d51f3f9,0000000..3bda9d6
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/security/handler/InsecurePermHandler.java
+++ b/server/src/main/java/org/apache/accumulo/server/security/handler/InsecurePermHandler.java
@@@ -1,146 -1,0 +1,104 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security.handler;
+
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+
+/**
+ * This is a Permission Handler implementation that doesn't actually do any security. Use at your own risk.
+ */
+public class InsecurePermHandler implements PermissionHandler {
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#initialize(java.lang.String)
- */
+ @Override
+ public void initialize(String instanceId, boolean initialize) {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#validSecurityHandlers(org.apache.accumulo.server.security.handler.Authenticator, org.apache.accumulo.server.security.handler.Authorizor)
- */
+ @Override
+ public boolean validSecurityHandlers(Authenticator authent, Authorizor author) {
+ return true;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#initializeSecurity(java.lang.String)
- */
+ @Override
+ public void initializeSecurity(TCredentials token, String rootuser) throws AccumuloSecurityException {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#hasSystemPermission(java.lang.String, org.apache.accumulo.core.security.SystemPermission)
- */
+ @Override
+ public boolean hasSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ return true;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#hasCachedSystemPermission(java.lang.String, org.apache.accumulo.core.security.SystemPermission)
- */
+ @Override
+ public boolean hasCachedSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ return true;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#hasTablePermission(java.lang.String, java.lang.String, org.apache.accumulo.core.security.TablePermission)
- */
+ @Override
+ public boolean hasTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ return true;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#hasCachedTablePermission(java.lang.String, java.lang.String, org.apache.accumulo.core.security.TablePermission)
- */
+ @Override
+ public boolean hasCachedTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ return true;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#grantSystemPermission(java.lang.String, org.apache.accumulo.core.security.SystemPermission)
- */
+ @Override
+ public void grantSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#revokeSystemPermission(java.lang.String, org.apache.accumulo.core.security.SystemPermission)
- */
+ @Override
+ public void revokeSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#grantTablePermission(java.lang.String, java.lang.String, org.apache.accumulo.core.security.TablePermission)
- */
+ @Override
+ public void grantTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#revokeTablePermission(java.lang.String, java.lang.String, org.apache.accumulo.core.security.TablePermission)
- */
+ @Override
+ public void revokeTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#cleanTablePermissions(java.lang.String)
- */
+ @Override
+ public void cleanTablePermissions(String table) throws AccumuloSecurityException, TableNotFoundException {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#initUser(java.lang.String)
- */
+ @Override
+ public void initUser(String user) throws AccumuloSecurityException {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#dropUser(java.lang.String)
- */
+ @Override
+ public void cleanUser(String user) throws AccumuloSecurityException {
+ return;
+ }
+
+ @Override
+ public void initTable(String table) throws AccumuloSecurityException {
+ }
+
+}
[22/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
index 36bd86a,0000000..8873938
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
+++ b/server/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
@@@ -1,157 -1,0 +1,156 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security.handler;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+public class ZKAuthorizor implements Authorizor {
+ private static final Logger log = Logger.getLogger(ZKAuthorizor.class);
+ private static Authorizor zkAuthorizorInstance = null;
+
+ private final String ZKUserAuths = "/Authorizations";
+
+ private String ZKUserPath;
+ private final ZooCache zooCache;
+
+ public static synchronized Authorizor getInstance() {
+ if (zkAuthorizorInstance == null)
+ zkAuthorizorInstance = new ZKAuthorizor();
+ return zkAuthorizorInstance;
+ }
+
+ public ZKAuthorizor() {
+ zooCache = new ZooCache();
+ }
+
++ @Override
+ public void initialize(String instanceId, boolean initialize) {
+ ZKUserPath = ZKSecurityTool.getInstancePath(instanceId) + "/users";
+ }
+
++ @Override
+ public Authorizations getCachedUserAuthorizations(String user) {
+ byte[] authsBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserAuths);
+ if (authsBytes != null)
+ return ZKSecurityTool.convertAuthorizations(authsBytes);
+ return Constants.NO_AUTHS;
+ }
+
+ @Override
+ public boolean validSecurityHandlers(Authenticator auth, PermissionHandler pm) {
+ return true;
+ }
+
+ @Override
+ public void initializeSecurity(TCredentials itw, String rootuser) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+
+ // create the root user with all system privileges, no table privileges, and no record-level authorizations
+ Set<SystemPermission> rootPerms = new TreeSet<SystemPermission>();
+ for (SystemPermission p : SystemPermission.values())
+ rootPerms.add(p);
+ Map<String,Set<TablePermission>> tablePerms = new HashMap<String,Set<TablePermission>>();
+ // Allow the root user to flush the !METADATA table
+ tablePerms.put(Constants.METADATA_TABLE_ID, Collections.singleton(TablePermission.ALTER_TABLE));
+
+ try {
+ // prep parent node of users with root username
+ if (!zoo.exists(ZKUserPath))
+ zoo.putPersistentData(ZKUserPath, rootuser.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
+
+ initUser(rootuser);
+ zoo.putPersistentData(ZKUserPath + "/" + rootuser + ZKUserAuths, ZKSecurityTool.convertAuthorizations(Constants.NO_AUTHS), NodeExistsPolicy.FAIL);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
- /**
- * @param user
- * @throws AccumuloSecurityException
- */
++ @Override
+ public void initUser(String user) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ try {
+ zoo.putPersistentData(ZKUserPath + "/" + user, new byte[0], NodeExistsPolicy.SKIP);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void dropUser(String user) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserAuths, NodeMissingPolicy.SKIP);
+ zooCache.clear(ZKUserPath + "/" + user);
+ }
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ if (e.code().equals(KeeperException.Code.NONODE))
+ throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+
+ }
+ }
+
+ @Override
+ public void changeAuthorizations(String user, Authorizations authorizations) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserAuths, ZKSecurityTool.convertAuthorizations(authorizations),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
index 4a05658,0000000..d802eb9
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
+++ b/server/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
@@@ -1,365 -1,0 +1,363 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security.handler;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.Code;
+
+/**
+ *
+ */
+public class ZKPermHandler implements PermissionHandler {
+ private static final Logger log = Logger.getLogger(ZKAuthorizor.class);
+ private static PermissionHandler zkPermHandlerInstance = null;
+
+ private String ZKUserPath;
+ private String ZKTablePath;
+ private final ZooCache zooCache;
+ private static final String ZKUserSysPerms = "/System";
+ private static final String ZKUserTablePerms = "/Tables";
+
+ public static synchronized PermissionHandler getInstance() {
+ if (zkPermHandlerInstance == null)
+ zkPermHandlerInstance = new ZKPermHandler();
+ return zkPermHandlerInstance;
+ }
+
++ @Override
+ public void initialize(String instanceId, boolean initialize) {
+ ZKUserPath = ZKSecurityTool.getInstancePath(instanceId) + "/users";
+ ZKTablePath = ZKSecurityTool.getInstancePath(instanceId) + "/tables";
+ }
+
+ public ZKPermHandler() {
+ zooCache = new ZooCache();
+ }
+
+ @Override
+ public boolean hasTablePermission(String user, String table, TablePermission permission) throws TableNotFoundException {
+ byte[] serializedPerms;
+ try {
+ String path = ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table;
+ ZooReaderWriter.getRetryingInstance().sync(path);
+ serializedPerms = ZooReaderWriter.getRetryingInstance().getData(path, null);
+ } catch (KeeperException e) {
+ if (e.code() == Code.NONODE) {
+ // maybe the table was just deleted?
+ try {
+ // check for existence:
+ ZooReaderWriter.getRetryingInstance().getData(ZKTablePath + "/" + table, null);
+ // it's there, you don't have permission
+ return false;
+ } catch (InterruptedException ex) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ } catch (KeeperException ex) {
+ // not there, throw an informative exception
+ if (e.code() == Code.NONODE) {
+ throw new TableNotFoundException(null, table, "while checking permissions");
+ }
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ }
+ return false;
+ }
+ log.warn("Unhandled KeeperException, failing closed for table permission check", e);
+ return false;
+ } catch (InterruptedException e) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ }
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertTablePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean hasCachedTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertTablePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public void grantSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ try {
+ byte[] permBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+ Set<SystemPermission> perms;
+ if (permBytes == null) {
+ perms = new TreeSet<SystemPermission>();
+ } else {
+ perms = ZKSecurityTool.convertSystemPermissions(permBytes);
+ }
+
+ if (perms.add(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(perms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void grantTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException {
+ Set<TablePermission> tablePerms;
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ if (serializedPerms != null)
+ tablePerms = ZKSecurityTool.convertTablePermissions(serializedPerms);
+ else
+ tablePerms = new TreeSet<TablePermission>();
+
+ try {
+ if (tablePerms.add(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, ZKSecurityTool.convertTablePermissions(tablePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void revokeSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] sysPermBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+
+ // User had no system permission, nothing to revoke.
+ if (sysPermBytes == null)
+ return;
+
+ Set<SystemPermission> sysPerms = ZKSecurityTool.convertSystemPermissions(sysPermBytes);
+
+ try {
+ if (sysPerms.remove(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(sysPerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void revokeTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+
+ // User had no table permission, nothing to revoke.
+ if (serializedPerms == null)
+ return;
+
+ Set<TablePermission> tablePerms = ZKSecurityTool.convertTablePermissions(serializedPerms);
+ try {
+ if (tablePerms.remove(permission)) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ if (tablePerms.size() == 0)
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, NodeMissingPolicy.SKIP);
+ else
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, ZKSecurityTool.convertTablePermissions(tablePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void cleanTablePermissions(String table) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ for (String user : zooCache.getChildren(ZKUserPath))
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, NodeMissingPolicy.SKIP);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException("unknownUser", SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void initializeSecurity(TCredentials itw, String rootuser) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+
+ // create the root user with all system privileges, no table privileges, and no record-level authorizations
+ Set<SystemPermission> rootPerms = new TreeSet<SystemPermission>();
+ for (SystemPermission p : SystemPermission.values())
+ rootPerms.add(p);
+ Map<String,Set<TablePermission>> tablePerms = new HashMap<String,Set<TablePermission>>();
+ // Allow the root user to flush the !METADATA table
+ tablePerms.put(Constants.METADATA_TABLE_ID, Collections.singleton(TablePermission.ALTER_TABLE));
+
+ try {
+ // prep parent node of users with root username
+ if (!zoo.exists(ZKUserPath))
+ zoo.putPersistentData(ZKUserPath, rootuser.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
+
+ initUser(rootuser);
+ zoo.putPersistentData(ZKUserPath + "/" + rootuser + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(rootPerms), NodeExistsPolicy.FAIL);
+ for (Entry<String,Set<TablePermission>> entry : tablePerms.entrySet())
+ createTablePerm(rootuser, entry.getKey(), entry.getValue());
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
- /**
- * @param user
- * @throws AccumuloSecurityException
- */
++ @Override
+ public void initUser(String user) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ try {
+ zoo.putPersistentData(ZKUserPath + "/" + user, new byte[0], NodeExistsPolicy.SKIP);
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms, new byte[0], NodeExistsPolicy.SKIP);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Sets up a new table configuration for the provided user/table. No checking for existence is done here, it should be done before calling.
+ */
+ private void createTablePerm(String user, String table, Set<TablePermission> perms) throws KeeperException, InterruptedException {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table,
+ ZKSecurityTool.convertTablePermissions(perms), NodeExistsPolicy.FAIL);
+ }
+ }
+
+ @Override
+ public void cleanUser(String user) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserSysPerms, NodeMissingPolicy.SKIP);
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms, NodeMissingPolicy.SKIP);
+ zooCache.clear(ZKUserPath + "/" + user);
+ }
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ if (e.code().equals(KeeperException.Code.NONODE))
+ throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+
+ }
+ }
+
+ @Override
+ public boolean hasSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] perms;
+ try {
+ String path = ZKUserPath + "/" + user + ZKUserSysPerms;
+ ZooReaderWriter.getRetryingInstance().sync(path);
+ perms = ZooReaderWriter.getRetryingInstance().getData(path, null);
+ } catch (KeeperException e) {
+ if (e.code() == Code.NONODE) {
+ return false;
+ }
+ log.warn("Unhandled KeeperException, failing closed for table permission check", e);
+ return false;
+ } catch (InterruptedException e) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ }
+
+ if (perms == null)
+ return false;
+ return ZKSecurityTool.convertSystemPermissions(perms).contains(permission);
+ }
+
+ @Override
+ public boolean hasCachedSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] perms = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+ if (perms == null)
+ return false;
+ return ZKSecurityTool.convertSystemPermissions(perms).contains(permission);
+ }
+
+ @Override
+ public boolean validSecurityHandlers(Authenticator authent, Authorizor author) {
+ return true;
+ }
+
+ @Override
+ public void initTable(String table) throws AccumuloSecurityException {
+ // All proper housekeeping is done on delete and permission granting, no work needs to be done here
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/tabletserver/MemValue.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/tabletserver/MemValue.java
index 735bf20,0000000..f68859a
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/MemValue.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/MemValue.java
@@@ -1,93 -1,0 +1,95 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.tabletserver;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.accumulo.core.data.Value;
+
+/**
+ *
+ */
+public class MemValue extends Value {
+ int kvCount;
+ boolean merged = false;
+
+ /**
+ * @param value
+ * Value
+ * @param kv
+ * kv count
+ */
+ public MemValue(byte[] value, int kv) {
+ super(value);
+ this.kvCount = kv;
+ }
+
+ public MemValue() {
+ super();
+ this.kvCount = Integer.MAX_VALUE;
+ }
+
+ public MemValue(Value value, int kv) {
+ super(value);
+ this.kvCount = kv;
+ }
+
+ // Override
++ @Override
+ public void write(final DataOutput out) throws IOException {
+ if (!merged) {
+ byte[] combinedBytes = new byte[getSize() + 4];
+ System.arraycopy(value, 0, combinedBytes, 4, getSize());
+ combinedBytes[0] = (byte) (kvCount >>> 24);
+ combinedBytes[1] = (byte) (kvCount >>> 16);
+ combinedBytes[2] = (byte) (kvCount >>> 8);
+ combinedBytes[3] = (byte) (kvCount);
+ value = combinedBytes;
+ merged = true;
+ }
+ super.write(out);
+ }
+
++ @Override
+ public void set(final byte[] b) {
+ super.set(b);
+ merged = false;
+ }
+
++ @Override
+ public void copy(byte[] b) {
+ super.copy(b);
+ merged = false;
+ }
+
+ /**
+ * Takes a Value and will take out the embedded kvCount, and then return that value while replacing the Value with the original unembedded version
+ *
- * @param v
+ * @return The kvCount embedded in v.
+ */
+ public static int splitKVCount(Value v) {
+ if (v instanceof MemValue)
+ return ((MemValue) v).kvCount;
+
+ byte[] originalBytes = new byte[v.getSize() - 4];
+ byte[] combined = v.get();
+ System.arraycopy(combined, 4, originalBytes, 0, originalBytes.length);
+ v.set(originalBytes);
+ return (combined[0] << 24) + ((combined[1] & 0xFF) << 16) + ((combined[2] & 0xFF) << 8) + (combined[3] & 0xFF);
+ }
+}
[60/64] [abbrv] Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
index cdcdfba,0000000..98d3c73
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
@@@ -1,276 -1,0 +1,272 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.metrics;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.lang.management.ManagementFactory;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+import javax.management.StandardMBean;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.commons.lang.builder.ToStringBuilder;
+import org.apache.commons.lang.time.DateUtils;
+
+public abstract class AbstractMetricsImpl {
+
+ public class Metric {
+
+ private long count = 0;
+ private long avg = 0;
+ private long min = 0;
+ private long max = 0;
+
+ public long getCount() {
+ return count;
+ }
+
+ public long getAvg() {
+ return avg;
+ }
+
+ public long getMin() {
+ return min;
+ }
+
+ public long getMax() {
+ return max;
+ }
+
+ public void incCount() {
+ count++;
+ }
+
+ public void addAvg(long a) {
+ if (a < 0)
+ return;
+ avg = (long) ((avg * .8) + (a * .2));
+ }
+
+ public void addMin(long a) {
+ if (a < 0)
+ return;
+ min = Math.min(min, a);
+ }
+
+ public void addMax(long a) {
+ if (a < 0)
+ return;
+ max = Math.max(max, a);
+ }
+
+ @Override
+ public String toString() {
+ return new ToStringBuilder(this).append("count", count).append("average", avg).append("minimum", min).append("maximum", max).toString();
+ }
+
+ }
+
+ static final org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger(AbstractMetricsImpl.class);
+
+ private static ConcurrentHashMap<String,Metric> registry = new ConcurrentHashMap<String,Metric>();
+
+ private boolean currentlyLogging = false;
+
+ private File logDir = null;
+
+ private String metricsPrefix = null;
+
+ private Date today = new Date();
+
+ private File logFile = null;
+
+ private Writer logWriter = null;
+
+ private SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd");
+
+ private SimpleDateFormat logFormatter = new SimpleDateFormat("yyyyMMddhhmmssz");
+
+ private MetricsConfiguration config = null;
+
+ public AbstractMetricsImpl() {
+ this.metricsPrefix = getMetricsPrefix();
+ config = new MetricsConfiguration(metricsPrefix);
+ }
+
+ /**
+ * Registers a StandardMBean with the MBean Server
- *
- * @throws Exception
+ */
+ public void register(StandardMBean mbean) throws Exception {
+ // Register this object with the MBeanServer
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ if (null == getObjectName())
+ throw new IllegalArgumentException("MBean object name must be set.");
+ mbs.registerMBean(mbean, getObjectName());
+
+ setupLogging();
+ }
+
+ /**
+ * Registers this MBean with the MBean Server
- *
- * @throws Exception
+ */
+ public void register() throws Exception {
+ // Register this object with the MBeanServer
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ if (null == getObjectName())
+ throw new IllegalArgumentException("MBean object name must be set.");
+ mbs.registerMBean(this, getObjectName());
+ setupLogging();
+ }
+
+ public void createMetric(String name) {
+ registry.put(name, new Metric());
+ }
+
+ public Metric getMetric(String name) {
+ return registry.get(name);
+ }
+
+ public long getMetricCount(String name) {
+ return registry.get(name).getCount();
+ }
+
+ public long getMetricAvg(String name) {
+ return registry.get(name).getAvg();
+ }
+
+ public long getMetricMin(String name) {
+ return registry.get(name).getMin();
+ }
+
+ public long getMetricMax(String name) {
+ return registry.get(name).getMax();
+ }
+
+ private void setupLogging() throws IOException {
+ if (null == config.getMetricsConfiguration())
+ return;
+ // If we are already logging, then return
+ if (!currentlyLogging && config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ // Check to see if directory exists, else make it
+ String mDir = config.getMetricsConfiguration().getString("logging.dir");
+ if (null != mDir) {
+ File dir = new File(mDir);
+ if (!dir.isDirectory())
+ if (!dir.mkdir())
+ log.warn("Could not create log directory: " + dir);
+ logDir = dir;
+ // Create new log file
+ startNewLog();
+ }
+ currentlyLogging = true;
+ }
+ }
+
+ private void startNewLog() throws IOException {
+ if (null != logWriter) {
+ logWriter.flush();
+ logWriter.close();
+ }
+ logFile = new File(logDir, metricsPrefix + "-" + formatter.format(today) + ".log");
+ if (!logFile.exists()) {
+ if (!logFile.createNewFile()) {
+ log.error("Unable to create new log file");
+ currentlyLogging = false;
+ return;
+ }
+ }
+ logWriter = new OutputStreamWriter(new FileOutputStream(logFile, true), Constants.UTF8);
+ }
+
+ private void writeToLog(String name) throws IOException {
+ if (null == logWriter)
+ return;
+ // Increment the date if we have to
+ Date now = new Date();
+ if (!DateUtils.isSameDay(today, now)) {
+ today = now;
+ startNewLog();
+ }
+ logWriter.append(logFormatter.format(now)).append(" Metric: ").append(name).append(": ").append(registry.get(name).toString()).append("\n");
+ }
+
+ public void add(String name, long time) {
+ if (isEnabled()) {
+ registry.get(name).incCount();
+ registry.get(name).addAvg(time);
+ registry.get(name).addMin(time);
+ registry.get(name).addMax(time);
+ // If we are not currently logging and should be, then initialize
+ if (!currentlyLogging && config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ try {
+ setupLogging();
+ } catch (IOException ioe) {
+ log.error("Error setting up log", ioe);
+ }
+ } else if (currentlyLogging && !config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ // if we are currently logging and shouldn't be, then close logs
+ try {
+ logWriter.flush();
+ logWriter.close();
+ logWriter = null;
+ logFile = null;
+ } catch (Exception e) {
+ log.error("Error stopping metrics logging", e);
+ }
+ currentlyLogging = false;
+ }
+ if (currentlyLogging) {
+ try {
+ writeToLog(name);
+ } catch (IOException ioe) {
+ log.error("Error writing to metrics log", ioe);
+ }
+ }
+ }
+ }
+
+ public boolean isEnabled() {
+ return config.isEnabled();
+ }
+
+ protected abstract ObjectName getObjectName();
+
+ protected abstract String getMetricsPrefix();
+
+ @Override
+ protected void finalize() {
+ if (null != logWriter) {
+ try {
+ logWriter.close();
+ } catch (Exception e) {
+ // do nothing
+ } finally {
+ logWriter = null;
+ }
+ }
+ logFile = null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
index 34d43f2,0000000..bbaf592
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
@@@ -1,175 -1,0 +1,171 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security.handler;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+public class ZKAuthorizor implements Authorizor {
+ private static final Logger log = Logger.getLogger(ZKAuthorizor.class);
+ private static Authorizor zkAuthorizorInstance = null;
+
+ private final String ZKUserAuths = "/Authorizations";
+
+ private String ZKUserPath;
+ private final ZooCache zooCache;
+
+ public static synchronized Authorizor getInstance() {
+ if (zkAuthorizorInstance == null)
+ zkAuthorizorInstance = new ZKAuthorizor();
+ return zkAuthorizorInstance;
+ }
+
+ public ZKAuthorizor() {
+ zooCache = new ZooCache();
+ }
+
+ @Override
+ public void initialize(String instanceId, boolean initialize) {
+ ZKUserPath = ZKSecurityTool.getInstancePath(instanceId) + "/users";
+ }
+
+ @Override
+ public Authorizations getCachedUserAuthorizations(String user) {
+ byte[] authsBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserAuths);
+ if (authsBytes != null)
+ return ZKSecurityTool.convertAuthorizations(authsBytes);
+ return Authorizations.EMPTY;
+ }
+
+ @Override
+ public boolean validSecurityHandlers(Authenticator auth, PermissionHandler pm) {
+ return true;
+ }
+
+ @Override
+ public void initializeSecurity(TCredentials itw, String rootuser) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+
+ // create the root user with all system privileges, no table privileges, and no record-level authorizations
+ Set<SystemPermission> rootPerms = new TreeSet<SystemPermission>();
+ for (SystemPermission p : SystemPermission.values())
+ rootPerms.add(p);
+ Map<String,Set<TablePermission>> tablePerms = new HashMap<String,Set<TablePermission>>();
+ // Allow the root user to flush the metadata tables
+ tablePerms.put(MetadataTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
+ tablePerms.put(RootTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
+
+ try {
+ // prep parent node of users with root username
+ if (!zoo.exists(ZKUserPath))
+ zoo.putPersistentData(ZKUserPath, rootuser.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
+
+ initUser(rootuser);
+ zoo.putPersistentData(ZKUserPath + "/" + rootuser + ZKUserAuths, ZKSecurityTool.convertAuthorizations(Authorizations.EMPTY), NodeExistsPolicy.FAIL);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
- /**
- * @param user
- * @throws AccumuloSecurityException
- */
+ @Override
+ public void initUser(String user) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ try {
+ zoo.putPersistentData(ZKUserPath + "/" + user, new byte[0], NodeExistsPolicy.SKIP);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void dropUser(String user) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserAuths, NodeMissingPolicy.SKIP);
+ zooCache.clear(ZKUserPath + "/" + user);
+ }
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ if (e.code().equals(KeeperException.Code.NONODE))
+ throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+
+ }
+ }
+
+ @Override
+ public void changeAuthorizations(String user, Authorizations authorizations) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserAuths, ZKSecurityTool.convertAuthorizations(authorizations),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean isValidAuthorizations(String user, List<ByteBuffer> auths) throws AccumuloSecurityException {
+ Collection<ByteBuffer> userauths = getCachedUserAuthorizations(user).getAuthorizationsBB();
+ for (ByteBuffer auth : auths)
+ if (!userauths.contains(auth))
+ return false;
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
index 6319653,0000000..1b7e7d3
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
@@@ -1,517 -1,0 +1,513 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security.handler;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.NamespacePermission;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.Code;
+
+/**
+ *
+ */
+public class ZKPermHandler implements PermissionHandler {
+ private static final Logger log = Logger.getLogger(ZKAuthorizor.class);
+ private static PermissionHandler zkPermHandlerInstance = null;
+
+ private String ZKUserPath;
+ private String ZKTablePath;
+ private String ZKNamespacePath;
+ private final ZooCache zooCache;
+ private final String ZKUserSysPerms = "/System";
+ private final String ZKUserTablePerms = "/Tables";
+ private final String ZKUserNamespacePerms = "/Namespaces";
+
+ public static synchronized PermissionHandler getInstance() {
+ if (zkPermHandlerInstance == null)
+ zkPermHandlerInstance = new ZKPermHandler();
+ return zkPermHandlerInstance;
+ }
+
+ @Override
+ public void initialize(String instanceId, boolean initialize) {
+ ZKUserPath = ZKSecurityTool.getInstancePath(instanceId) + "/users";
+ ZKTablePath = ZKSecurityTool.getInstancePath(instanceId) + "/tables";
+ ZKNamespacePath = ZKSecurityTool.getInstancePath(instanceId) + "/namespaces";
+ }
+
+ public ZKPermHandler() {
+ zooCache = new ZooCache();
+ }
+
+ @Override
+ public boolean hasTablePermission(String user, String table, TablePermission permission) throws TableNotFoundException {
+ byte[] serializedPerms;
+ try {
+ String path = ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table;
+ ZooReaderWriter.getRetryingInstance().sync(path);
+ serializedPerms = ZooReaderWriter.getRetryingInstance().getData(path, null);
+ } catch (KeeperException e) {
+ if (e.code() == Code.NONODE) {
+ // maybe the table was just deleted?
+ try {
+ // check for existence:
+ ZooReaderWriter.getRetryingInstance().getData(ZKTablePath + "/" + table, null);
+ // it's there, you don't have permission
+ return false;
+ } catch (InterruptedException ex) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ } catch (KeeperException ex) {
+ // not there, throw an informative exception
+ if (e.code() == Code.NONODE) {
+ throw new TableNotFoundException(null, table, "while checking permissions");
+ }
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ }
+ return false;
+ }
+ log.warn("Unhandled KeeperException, failing closed for table permission check", e);
+ return false;
+ } catch (InterruptedException e) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ }
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertTablePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean hasCachedTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertTablePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean hasNamespacePermission(String user, String namespace, NamespacePermission permission) throws NamespaceNotFoundException {
+ byte[] serializedPerms;
+ try {
+ String path = ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace;
+ ZooReaderWriter.getRetryingInstance().sync(path);
+ serializedPerms = ZooReaderWriter.getRetryingInstance().getData(path, null);
+ } catch (KeeperException e) {
+ if (e.code() == Code.NONODE) {
+ // maybe the namespace was just deleted?
+ try {
+ // check for existence:
+ ZooReaderWriter.getRetryingInstance().getData(ZKNamespacePath + "/" + namespace, null);
+ // it's there, you don't have permission
+ return false;
+ } catch (InterruptedException ex) {
+ log.warn("Unhandled InterruptedException, failing closed for namespace permission check", e);
+ return false;
+ } catch (KeeperException ex) {
+ // not there, throw an informative exception
+ if (e.code() == Code.NONODE) {
+ throw new NamespaceNotFoundException(null, namespace, "while checking permissions");
+ }
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ }
+ return false;
+ }
+ log.warn("Unhandled KeeperException, failing closed for table permission check", e);
+ return false;
+ } catch (InterruptedException e) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ }
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertNamespacePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean hasCachedNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException,
+ NamespaceNotFoundException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertNamespacePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public void grantSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ try {
+ byte[] permBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+ Set<SystemPermission> perms;
+ if (permBytes == null) {
+ perms = new TreeSet<SystemPermission>();
+ } else {
+ perms = ZKSecurityTool.convertSystemPermissions(permBytes);
+ }
+
+ if (perms.add(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(perms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void grantTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException {
+ Set<TablePermission> tablePerms;
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ if (serializedPerms != null)
+ tablePerms = ZKSecurityTool.convertTablePermissions(serializedPerms);
+ else
+ tablePerms = new TreeSet<TablePermission>();
+
+ try {
+ if (tablePerms.add(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, ZKSecurityTool.convertTablePermissions(tablePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void grantNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException {
+ Set<NamespacePermission> namespacePerms;
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
+ if (serializedPerms != null)
+ namespacePerms = ZKSecurityTool.convertNamespacePermissions(serializedPerms);
+ else
+ namespacePerms = new TreeSet<NamespacePermission>();
+
+ try {
+ if (namespacePerms.add(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, ZKSecurityTool.convertNamespacePermissions(namespacePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void revokeSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] sysPermBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+
+ // User had no system permission, nothing to revoke.
+ if (sysPermBytes == null)
+ return;
+
+ Set<SystemPermission> sysPerms = ZKSecurityTool.convertSystemPermissions(sysPermBytes);
+
+ try {
+ if (sysPerms.remove(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(sysPerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void revokeTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+
+ // User had no table permission, nothing to revoke.
+ if (serializedPerms == null)
+ return;
+
+ Set<TablePermission> tablePerms = ZKSecurityTool.convertTablePermissions(serializedPerms);
+ try {
+ if (tablePerms.remove(permission)) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ if (tablePerms.size() == 0)
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, NodeMissingPolicy.SKIP);
+ else
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, ZKSecurityTool.convertTablePermissions(tablePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void revokeNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
+
+ // User had no namespace permission, nothing to revoke.
+ if (serializedPerms == null)
+ return;
+
+ Set<NamespacePermission> namespacePerms = ZKSecurityTool.convertNamespacePermissions(serializedPerms);
+ try {
+ if (namespacePerms.remove(permission)) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ if (namespacePerms.size() == 0)
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, NodeMissingPolicy.SKIP);
+ else
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, ZKSecurityTool.convertNamespacePermissions(namespacePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void cleanTablePermissions(String table) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ for (String user : zooCache.getChildren(ZKUserPath))
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, NodeMissingPolicy.SKIP);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException("unknownUser", SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void cleanNamespacePermissions(String namespace) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ for (String user : zooCache.getChildren(ZKUserPath))
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, NodeMissingPolicy.SKIP);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException("unknownUser", SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void initializeSecurity(TCredentials itw, String rootuser) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+
+ // create the root user with all system privileges, no table privileges, and no record-level authorizations
+ Set<SystemPermission> rootPerms = new TreeSet<SystemPermission>();
+ for (SystemPermission p : SystemPermission.values())
+ rootPerms.add(p);
+ Map<String,Set<TablePermission>> tablePerms = new HashMap<String,Set<TablePermission>>();
+ // Allow the root user to flush the system tables
+ tablePerms.put(RootTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
+ tablePerms.put(MetadataTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
+ // essentially the same but on the system namespace, the ALTER_TABLE permission is now redundant
+ Map<String,Set<NamespacePermission>> namespacePerms = new HashMap<String,Set<NamespacePermission>>();
+ namespacePerms.put(Namespaces.ACCUMULO_NAMESPACE_ID, Collections.singleton(NamespacePermission.ALTER_NAMESPACE));
+ namespacePerms.put(Namespaces.ACCUMULO_NAMESPACE_ID, Collections.singleton(NamespacePermission.ALTER_TABLE));
+
+ try {
+ // prep parent node of users with root username
+ if (!zoo.exists(ZKUserPath))
+ zoo.putPersistentData(ZKUserPath, rootuser.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
+
+ initUser(rootuser);
+ zoo.putPersistentData(ZKUserPath + "/" + rootuser + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(rootPerms), NodeExistsPolicy.FAIL);
+ for (Entry<String,Set<TablePermission>> entry : tablePerms.entrySet())
+ createTablePerm(rootuser, entry.getKey(), entry.getValue());
+ for (Entry<String,Set<NamespacePermission>> entry : namespacePerms.entrySet())
+ createNamespacePerm(rootuser, entry.getKey(), entry.getValue());
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
- /**
- * @param user
- * @throws AccumuloSecurityException
- */
+ @Override
+ public void initUser(String user) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ try {
+ zoo.putPersistentData(ZKUserPath + "/" + user, new byte[0], NodeExistsPolicy.SKIP);
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms, new byte[0], NodeExistsPolicy.SKIP);
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms, new byte[0], NodeExistsPolicy.SKIP);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Sets up a new table configuration for the provided user/table. No checking for existence is done here, it should be done before calling.
+ */
+ private void createTablePerm(String user, String table, Set<TablePermission> perms) throws KeeperException, InterruptedException {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table,
+ ZKSecurityTool.convertTablePermissions(perms), NodeExistsPolicy.FAIL);
+ }
+ }
+
+ /**
+ * Sets up a new namespace configuration for the provided user/table. No checking for existence is done here, it should be done before calling.
+ */
+ private void createNamespacePerm(String user, String namespace, Set<NamespacePermission> perms) throws KeeperException, InterruptedException {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace,
+ ZKSecurityTool.convertNamespacePermissions(perms), NodeExistsPolicy.FAIL);
+ }
+ }
+
+ @Override
+ public void cleanUser(String user) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserSysPerms, NodeMissingPolicy.SKIP);
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms, NodeMissingPolicy.SKIP);
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms, NodeMissingPolicy.SKIP);
+ zooCache.clear(ZKUserPath + "/" + user);
+ }
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ if (e.code().equals(KeeperException.Code.NONODE))
+ throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+
+ }
+ }
+
+ @Override
+ public boolean hasSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] perms;
+ try {
+ String path = ZKUserPath + "/" + user + ZKUserSysPerms;
+ ZooReaderWriter.getRetryingInstance().sync(path);
+ perms = ZooReaderWriter.getRetryingInstance().getData(path, null);
+ } catch (KeeperException e) {
+ if (e.code() == Code.NONODE) {
+ return false;
+ }
+ log.warn("Unhandled KeeperException, failing closed for table permission check", e);
+ return false;
+ } catch (InterruptedException e) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ }
+
+ if (perms == null)
+ return false;
+ return ZKSecurityTool.convertSystemPermissions(perms).contains(permission);
+ }
+
+ @Override
+ public boolean hasCachedSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] perms = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+ if (perms == null)
+ return false;
+ return ZKSecurityTool.convertSystemPermissions(perms).contains(permission);
+ }
+
+ @Override
+ public boolean validSecurityHandlers(Authenticator authent, Authorizor author) {
+ return true;
+ }
+
+ @Override
+ public void initTable(String table) throws AccumuloSecurityException {
+ // All proper housekeeping is done on delete and permission granting, no work needs to be done here
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
index e16bd06,0000000..cf1a065
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
@@@ -1,62 -1,0 +1,59 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.TokenProperty;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.security.handler.Authenticator;
+import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+
+/**
+ *
+ */
+public class LoginProperties {
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ AccumuloConfiguration config = ServerConfiguration.getSystemConfiguration(HdfsZooInstance.getInstance());
+ Authenticator authenticator = AccumuloClassLoader.getClassLoader().loadClass(config.get(Property.INSTANCE_SECURITY_AUTHENTICATOR))
+ .asSubclass(Authenticator.class).newInstance();
+
+ List<Set<TokenProperty>> tokenProps = new ArrayList<Set<TokenProperty>>();
+
+ for (Class<? extends AuthenticationToken> tokenType : authenticator.getSupportedTokenTypes()) {
+ tokenProps.add(tokenType.newInstance().getProperties());
+ }
+
+ System.out.println("Supported token types for " + authenticator.getClass().getName() + " are : ");
+ for (Class<? extends AuthenticationToken> tokenType : authenticator.getSupportedTokenTypes()) {
+ System.out.println("\t" + tokenType.getName() + ", which accepts the following properties : ");
+
+ for (TokenProperty tokenProperty : tokenType.newInstance().getProperties()) {
+ System.out.println("\t\t" + tokenProperty);
+ }
+
+ System.out.println();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
index 6e5607e,0000000..37ef5f1
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
@@@ -1,128 -1,0 +1,124 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.Stack;
+
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.DefaultHandler;
+
+import com.beust.jcommander.Parameter;
+
+public class RestoreZookeeper {
+
+ private static class Restore extends DefaultHandler {
+ IZooReaderWriter zk = null;
+ Stack<String> cwd = new Stack<String>();
+ boolean overwrite = false;
+
+ Restore(IZooReaderWriter zk, boolean overwrite) {
+ this.zk = zk;
+ this.overwrite = overwrite;
+ }
+
+ @Override
+ public void startElement(String uri, String localName, String name, Attributes attributes) throws SAXException {
+ if ("node".equals(name)) {
+ String child = attributes.getValue("name");
+ if (child == null)
+ throw new RuntimeException("name attribute not set");
+ String encoding = attributes.getValue("encoding");
+ String value = attributes.getValue("value");
+ if (value == null)
+ value = "";
+ String path = cwd.lastElement() + "/" + child;
+ create(path, value, encoding);
+ cwd.push(path);
+ } else if ("dump".equals(name)) {
+ String root = attributes.getValue("root");
+ if (root.equals("/"))
+ cwd.push("");
+ else
+ cwd.push(root);
+ create(root, "", Constants.UTF8.name());
+ }
+ }
+
+ @Override
+ public void endElement(String uri, String localName, String name) throws SAXException {
+ cwd.pop();
+ }
+
+ // assume UTF-8 if not "base64"
+ private void create(String path, String value, String encoding) {
+ byte[] data = value.getBytes(Constants.UTF8);
+ if ("base64".equals(encoding))
+ data = Base64.decodeBase64(data);
+ try {
+ try {
+ zk.putPersistentData(path, data, overwrite ? NodeExistsPolicy.OVERWRITE : NodeExistsPolicy.FAIL);
+ } catch (KeeperException e) {
+ if (e.code().equals(KeeperException.Code.NODEEXISTS))
+ throw new RuntimeException(path + " exists. Remove it first.");
+ throw e;
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static class Opts extends Help {
+ @Parameter(names = {"-z", "--keepers"})
+ String keepers = "localhost:2181";
+ @Parameter(names = "--overwrite")
+ boolean overwrite = false;
+ @Parameter(names = "--file")
+ String file;
+ }
+
- /**
- * @param args
- * @throws Exception
- */
+ public static void main(String[] args) throws Exception {
+ Logger.getRootLogger().setLevel(Level.WARN);
+ Opts opts = new Opts();
+ opts.parseArgs(RestoreZookeeper.class.getName(), args);
+
+ InputStream in = System.in;
+ if (opts.file != null) {
+ in = new FileInputStream(opts.file);
+ }
+
+ SAXParserFactory factory = SAXParserFactory.newInstance();
+ SAXParser parser = factory.newSAXParser();
+ parser.parse(in, new Restore(ZooReaderWriter.getInstance(), opts.overwrite));
+ in.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
index cb932d7,0000000..9a54927
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
@@@ -1,290 -1,0 +1,287 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.NumUtil;
+import org.apache.accumulo.core.util.StringUtil;
+import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+public class TableDiskUsage {
+
+ private static final Logger log = Logger.getLogger(Logger.class);
+ private int nextInternalId = 0;
+ private Map<String,Integer> internalIds = new HashMap<String,Integer>();
+ private Map<Integer,String> externalIds = new HashMap<Integer,String>();
+ private Map<String,Integer[]> tableFiles = new HashMap<String,Integer[]>();
+ private Map<String,Long> fileSizes = new HashMap<String,Long>();
+
+ void addTable(String tableId) {
+ if (internalIds.containsKey(tableId))
+ throw new IllegalArgumentException("Already added table " + tableId);
+
+ int iid = nextInternalId++;
+
+ internalIds.put(tableId, iid);
+ externalIds.put(iid, tableId);
+ }
+
+ void linkFileAndTable(String tableId, String file) {
+ int internalId = internalIds.get(tableId);
+
+ Integer[] tables = tableFiles.get(file);
+ if (tables == null) {
+ tables = new Integer[internalIds.size()];
+ for (int i = 0; i < tables.length; i++)
+ tables[i] = 0;
+ tableFiles.put(file, tables);
+ }
+
+ tables[internalId] = 1;
+ }
+
+ void addFileSize(String file, long size) {
+ fileSizes.put(file, size);
+ }
+
+ Map<List<String>,Long> calculateUsage() {
+
+ Map<List<Integer>,Long> usage = new HashMap<List<Integer>,Long>();
+
+ for (Entry<String,Integer[]> entry : tableFiles.entrySet()) {
+ log.info("fileSizes " + fileSizes + " key " + Arrays.asList(entry.getKey()));
+ List<Integer> key = Arrays.asList(entry.getValue());
+ Long size = fileSizes.get(entry.getKey());
+
+ Long tablesUsage = usage.get(key);
+ if (tablesUsage == null)
+ tablesUsage = 0l;
+
+ tablesUsage += size;
+
+ usage.put(key, tablesUsage);
+
+ }
+
+ Map<List<String>,Long> externalUsage = new HashMap<List<String>,Long>();
+
+ for (Entry<List<Integer>,Long> entry : usage.entrySet()) {
+ List<String> externalKey = new ArrayList<String>();
+ List<Integer> key = entry.getKey();
+ for (int i = 0; i < key.size(); i++)
+ if (key.get(i) != 0)
+ externalKey.add(externalIds.get(i));
+
+ externalUsage.put(externalKey, entry.getValue());
+ }
+
+ return externalUsage;
+ }
+
+ public interface Printer {
+ void print(String line);
+ }
+
+ public static void printDiskUsage(AccumuloConfiguration acuConf, Collection<String> tables, VolumeManager fs, Connector conn, boolean humanReadable)
+ throws TableNotFoundException, IOException {
+ printDiskUsage(acuConf, tables, fs, conn, new Printer() {
+ @Override
+ public void print(String line) {
+ System.out.println(line);
+ }
+ }, humanReadable);
+ }
+
+ public static Map<TreeSet<String>,Long> getDiskUsage(AccumuloConfiguration acuConf, Set<String> tableIds, VolumeManager fs, Connector conn)
+ throws IOException {
+ TableDiskUsage tdu = new TableDiskUsage();
+
+ for (String tableId : tableIds)
+ tdu.addTable(tableId);
+
+ HashSet<String> tablesReferenced = new HashSet<String>(tableIds);
+ HashSet<String> emptyTableIds = new HashSet<String>();
+ HashSet<String> nameSpacesReferenced = new HashSet<String>();
+
+ for (String tableId : tableIds) {
+ Scanner mdScanner = null;
+ try {
+ mdScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ } catch (TableNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ mdScanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+
+ if (!mdScanner.iterator().hasNext()) {
+ emptyTableIds.add(tableId);
+ }
+
+ for (Entry<Key,Value> entry : mdScanner) {
+ String file = entry.getKey().getColumnQualifier().toString();
+ String parts[] = file.split("/");
+ String uniqueName = parts[parts.length - 1];
+ if (file.contains(":") || file.startsWith("../")) {
+ String ref = parts[parts.length - 3];
+ if (!ref.equals(tableId)) {
+ tablesReferenced.add(ref);
+ }
+ if (file.contains(":") && parts.length > 3) {
+ List<String> base = Arrays.asList(Arrays.copyOf(parts, parts.length - 3));
+ nameSpacesReferenced.add(StringUtil.join(base, "/"));
+ }
+ }
+
+ tdu.linkFileAndTable(tableId, uniqueName);
+ }
+ }
+
+ for (String tableId : tablesReferenced) {
+ for (String tableDir : nameSpacesReferenced) {
+ FileStatus[] files = fs.globStatus(new Path(tableDir + "/" + tableId + "/*/*"));
+ if (files != null) {
+ for (FileStatus fileStatus : files) {
+ // Assumes that all filenames are unique
+ String name = fileStatus.getPath().getName();
+ tdu.addFileSize(name, fileStatus.getLen());
+ }
+ }
+ }
+ }
+
+ HashMap<String,String> reverseTableIdMap = new HashMap<String,String>();
+ for (Entry<String,String> entry : conn.tableOperations().tableIdMap().entrySet())
+ reverseTableIdMap.put(entry.getValue(), entry.getKey());
+
+ TreeMap<TreeSet<String>,Long> usage = new TreeMap<TreeSet<String>,Long>(new Comparator<TreeSet<String>>() {
+
+ @Override
+ public int compare(TreeSet<String> o1, TreeSet<String> o2) {
+ int len1 = o1.size();
+ int len2 = o2.size();
+
+ int min = Math.min(len1, len2);
+
+ Iterator<String> iter1 = o1.iterator();
+ Iterator<String> iter2 = o2.iterator();
+
+ int count = 0;
+
+ while (count < min) {
+ String s1 = iter1.next();
+ String s2 = iter2.next();
+
+ int cmp = s1.compareTo(s2);
+
+ if (cmp != 0)
+ return cmp;
+
+ count++;
+ }
+
+ return len1 - len2;
+ }
+ });
+
+ for (Entry<List<String>,Long> entry : tdu.calculateUsage().entrySet()) {
+ TreeSet<String> tableNames = new TreeSet<String>();
+ for (String tableId : entry.getKey())
+ tableNames.add(reverseTableIdMap.get(tableId));
+
+ usage.put(tableNames, entry.getValue());
+ }
+
+ if (!emptyTableIds.isEmpty()) {
+ TreeSet<String> emptyTables = new TreeSet<String>();
+ for (String tableId : emptyTableIds) {
+ emptyTables.add(reverseTableIdMap.get(tableId));
+ }
+ usage.put(emptyTables, 0L);
+ }
+
+ return usage;
+ }
+
+ public static void printDiskUsage(AccumuloConfiguration acuConf, Collection<String> tables, VolumeManager fs, Connector conn, Printer printer,
+ boolean humanReadable) throws TableNotFoundException, IOException {
+
+ HashSet<String> tableIds = new HashSet<String>();
+
+ for (String tableName : tables) {
+ String tableId = conn.tableOperations().tableIdMap().get(tableName);
+ if (tableId == null)
+ throw new TableNotFoundException(null, tableName, "Table " + tableName + " not found");
+
+ tableIds.add(tableId);
+ }
+
+ Map<TreeSet<String>,Long> usage = getDiskUsage(acuConf, tableIds, fs, conn);
+
+ String valueFormat = humanReadable ? "%9s" : "%,24d";
+ for (Entry<TreeSet<String>,Long> entry : usage.entrySet()) {
+ Object value = humanReadable ? NumUtil.bigNumberForSize(entry.getValue()) : entry.getValue();
+ printer.print(String.format(valueFormat + " %s", value, entry.getKey()));
+ }
+ }
+
+ static class Opts extends ClientOpts {
+ @Parameter(description = " <table> { <table> ... } ")
+ List<String> tables = new ArrayList<String>();
+ }
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ VolumeManager fs = VolumeManagerImpl.get();
+ Opts opts = new Opts();
+ opts.parseArgs(TableDiskUsage.class.getName(), args);
+ Connector conn = opts.getConnector();
+ org.apache.accumulo.server.util.TableDiskUsage.printDiskUsage(DefaultConfiguration.getInstance(), opts.tables, fs, conn, false);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
index 2fc0bd3,0000000..34c2151
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
@@@ -1,75 -1,0 +1,72 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.util.List;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+
+import com.beust.jcommander.Parameter;
+
+public class TabletServerLocks {
+
+ static class Opts extends Help {
+ @Parameter(names="-list")
+ boolean list = false;
+ @Parameter(names="-delete")
+ String delete = null;
+ }
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+
+ Instance instance = HdfsZooInstance.getInstance();
+ String tserverPath = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
+ Opts opts = new Opts();
+ opts.parseArgs(TabletServerLocks.class.getName(), args);
+
+ ZooCache cache = new ZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
+
+ if (opts.list) {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ List<String> tabletServers = zoo.getChildren(tserverPath);
+
+ for (String tabletServer : tabletServers) {
+ byte[] lockData = ZooLock.getLockData(cache, tserverPath + "/" + tabletServer, null);
+ String holder = null;
+ if (lockData != null) {
+ holder = new String(lockData, Constants.UTF8);
+ }
+
+ System.out.printf("%32s %16s%n", tabletServer, holder);
+ }
+ } else if (opts.delete != null) {
+ ZooLock.deleteLock(tserverPath + "/" + args[1]);
+ } else {
+ System.out.println("Usage : " + TabletServerLocks.class.getName() + " -list|-delete <tserver lock>");
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/tserver/src/main/java/org/apache/accumulo/tserver/MemValue.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/MemValue.java
index 13bcdbe,0000000..f1fdde4
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/MemValue.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/MemValue.java
@@@ -1,93 -1,0 +1,95 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.accumulo.core.data.Value;
+
+/**
+ *
+ */
+public class MemValue extends Value {
+ int kvCount;
+ boolean merged = false;
+
+ /**
+ * @param value
+ * Value
+ * @param kv
+ * kv count
+ */
+ public MemValue(byte[] value, int kv) {
+ super(value);
+ this.kvCount = kv;
+ }
+
+ public MemValue() {
+ super();
+ this.kvCount = Integer.MAX_VALUE;
+ }
+
+ public MemValue(Value value, int kv) {
+ super(value);
+ this.kvCount = kv;
+ }
+
+ // Override
++ @Override
+ public void write(final DataOutput out) throws IOException {
+ if (!merged) {
+ byte[] combinedBytes = new byte[getSize() + 4];
+ System.arraycopy(value, 0, combinedBytes, 4, getSize());
+ combinedBytes[0] = (byte) (kvCount >>> 24);
+ combinedBytes[1] = (byte) (kvCount >>> 16);
+ combinedBytes[2] = (byte) (kvCount >>> 8);
+ combinedBytes[3] = (byte) (kvCount);
+ value = combinedBytes;
+ merged = true;
+ }
+ super.write(out);
+ }
+
++ @Override
+ public void set(final byte[] b) {
+ super.set(b);
+ merged = false;
+ }
+
++ @Override
+ public void copy(byte[] b) {
+ super.copy(b);
+ merged = false;
+ }
+
+ /**
+ * Takes a Value and will take out the embedded kvCount, and then return that value while replacing the Value with the original unembedded version
+ *
- * @param v
+ * @return The kvCount embedded in v.
+ */
+ public static int splitKVCount(Value v) {
+ if (v instanceof MemValue)
+ return ((MemValue) v).kvCount;
+
+ byte[] originalBytes = new byte[v.getSize() - 4];
+ byte[] combined = v.get();
+ System.arraycopy(combined, 4, originalBytes, 0, originalBytes.length);
+ v.set(originalBytes);
+ return (combined[0] << 24) + ((combined[1] & 0xFF) << 16) + ((combined[2] & 0xFF) << 8) + (combined[3] & 0xFF);
+ }
+}
[31/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftTransportPool.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/ThriftTransportPool.java
index f123289,0000000..d2113fb
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftTransportPool.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftTransportPool.java
@@@ -1,671 -1,0 +1,683 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.impl;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.security.SecurityPermission;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.TTimeoutTransport;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.log4j.Logger;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+
+public class ThriftTransportPool {
+ private static SecurityPermission TRANSPORT_POOL_PERMISSION = new SecurityPermission("transportPoolPermission");
+
+ private static final Random random = new Random();
+ private long killTime = 1000 * 3;
+
+ private Map<ThriftTransportKey,List<CachedConnection>> cache = new HashMap<ThriftTransportKey,List<CachedConnection>>();
+ private Map<ThriftTransportKey,Long> errorCount = new HashMap<ThriftTransportKey,Long>();
+ private Map<ThriftTransportKey,Long> errorTime = new HashMap<ThriftTransportKey,Long>();
+ private Set<ThriftTransportKey> serversWarnedAbout = new HashSet<ThriftTransportKey>();
+
+ private CountDownLatch closerExitLatch;
+
+ private static final Logger log = Logger.getLogger(ThriftTransportPool.class);
+
+ private static final Long ERROR_THRESHOLD = 20l;
+ private static final int STUCK_THRESHOLD = 2 * 60 * 1000;
+
+ private static class CachedConnection {
+
+ public CachedConnection(CachedTTransport t) {
+ this.transport = t;
+ }
+
+ void setReserved(boolean reserved) {
+ this.transport.setReserved(reserved);
+ }
+
+ boolean isReserved() {
+ return this.transport.reserved;
+ }
+
+ CachedTTransport transport;
+
+ long lastReturnTime;
+ }
+
+ public static class TransportPoolShutdownException extends RuntimeException {
+ private static final long serialVersionUID = 1L;
+ }
+
+ private static class Closer implements Runnable {
+ final ThriftTransportPool pool;
+ private CountDownLatch closerExitLatch;
+
+ public Closer(ThriftTransportPool pool, CountDownLatch closerExitLatch) {
+ this.pool = pool;
+ this.closerExitLatch = closerExitLatch;
+ }
+
+ private void closeConnections() {
+ while (true) {
+
+ ArrayList<CachedConnection> connectionsToClose = new ArrayList<CachedConnection>();
+
+ synchronized (pool) {
+ for (List<CachedConnection> ccl : pool.getCache().values()) {
+ Iterator<CachedConnection> iter = ccl.iterator();
+ while (iter.hasNext()) {
+ CachedConnection cachedConnection = iter.next();
+
+ if (!cachedConnection.isReserved() && System.currentTimeMillis() - cachedConnection.lastReturnTime > pool.killTime) {
+ connectionsToClose.add(cachedConnection);
+ iter.remove();
+ }
+ }
+ }
+
+ for (List<CachedConnection> ccl : pool.getCache().values()) {
+ for (CachedConnection cachedConnection : ccl) {
+ cachedConnection.transport.checkForStuckIO(STUCK_THRESHOLD);
+ }
+ }
+
+ Iterator<Entry<ThriftTransportKey,Long>> iter = pool.errorTime.entrySet().iterator();
+ while (iter.hasNext()) {
+ Entry<ThriftTransportKey,Long> entry = iter.next();
+ long delta = System.currentTimeMillis() - entry.getValue();
+ if (delta >= STUCK_THRESHOLD) {
+ pool.errorCount.remove(entry.getKey());
+ iter.remove();
+ }
+ }
+ }
+
+ // close connections outside of sync block
+ for (CachedConnection cachedConnection : connectionsToClose) {
+ cachedConnection.transport.close();
+ }
+
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
++ @Override
+ public void run() {
+ try {
+ closeConnections();
+ } catch (TransportPoolShutdownException e) {
+ } finally {
+ closerExitLatch.countDown();
+ }
+ }
+ }
+
+ static class CachedTTransport extends TTransport {
+
+ private ThriftTransportKey cacheKey;
+ private TTransport wrappedTransport;
+ private boolean sawError = false;
+
+ private volatile String ioThreadName = null;
+ private volatile long ioStartTime = 0;
+ private volatile boolean reserved = false;
+
+ private String stuckThreadName = null;
+
+ int ioCount = 0;
+ int lastIoCount = -1;
+
+ private void sawError(Exception e) {
+ sawError = true;
+ }
+
+ final void setReserved(boolean reserved) {
+ this.reserved = reserved;
+ if (reserved) {
+ ioThreadName = Thread.currentThread().getName();
+ ioCount = 0;
+ lastIoCount = -1;
+ } else {
+ if ((ioCount & 1) == 1) {
+ // connection unreserved, but it seems io may still be
+ // happening
+ log.warn("Connection returned to thrift connection pool that may still be in use " + ioThreadName + " " + Thread.currentThread().getName(),
+ new Exception());
+ }
+
+ ioCount = 0;
+ lastIoCount = -1;
+ ioThreadName = null;
+ }
+ checkForStuckIO(STUCK_THRESHOLD);
+ }
+
+ final void checkForStuckIO(long threshold) {
+ /*
+ * checking for stuck io needs to be light weight.
+ *
+ * Tried to call System.currentTimeMillis() and Thread.currentThread() before every io operation.... this dramatically slowed things down. So switched to
+ * incrementing a counter before and after each io operation.
+ */
+
+ if ((ioCount & 1) == 1) {
+ // when ioCount is odd, it means I/O is currently happening
+ if (ioCount == lastIoCount) {
+ // still doing same I/O operation as last time this
+ // functions was called
+ long delta = System.currentTimeMillis() - ioStartTime;
+ if (delta >= threshold && stuckThreadName == null) {
+ stuckThreadName = ioThreadName;
+ log.warn("Thread \"" + ioThreadName + "\" stuck on IO to " + cacheKey + " for at least " + delta + " ms");
+ }
+ } else {
+ // remember this ioCount and the time we saw it, need to see
+ // if it changes
+ lastIoCount = ioCount;
+ ioStartTime = System.currentTimeMillis();
+
+ if (stuckThreadName != null) {
+ // doing I/O, but ioCount changed so no longer stuck
+ log.info("Thread \"" + stuckThreadName + "\" no longer stuck on IO to " + cacheKey + " sawError = " + sawError);
+ stuckThreadName = null;
+ }
+ }
+ } else {
+ // I/O is not currently happening
+ if (stuckThreadName != null) {
+ // no longer stuck, and was stuck in the past
+ log.info("Thread \"" + stuckThreadName + "\" no longer stuck on IO to " + cacheKey + " sawError = " + sawError);
+ stuckThreadName = null;
+ }
+ }
+ }
+
+ public CachedTTransport(TTransport transport, ThriftTransportKey cacheKey2) {
+ this.wrappedTransport = transport;
+ this.cacheKey = cacheKey2;
+ }
+
++ @Override
+ public boolean isOpen() {
+ return wrappedTransport.isOpen();
+ }
+
++ @Override
+ public void open() throws TTransportException {
+ try {
+ ioCount++;
+ wrappedTransport.open();
+ } catch (TTransportException tte) {
+ sawError(tte);
+ throw tte;
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public int read(byte[] arg0, int arg1, int arg2) throws TTransportException {
+ try {
+ ioCount++;
+ return wrappedTransport.read(arg0, arg1, arg2);
+ } catch (TTransportException tte) {
+ sawError(tte);
+ throw tte;
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public int readAll(byte[] arg0, int arg1, int arg2) throws TTransportException {
+ try {
+ ioCount++;
+ return wrappedTransport.readAll(arg0, arg1, arg2);
+ } catch (TTransportException tte) {
+ sawError(tte);
+ throw tte;
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public void write(byte[] arg0, int arg1, int arg2) throws TTransportException {
+ try {
+ ioCount++;
+ wrappedTransport.write(arg0, arg1, arg2);
+ } catch (TTransportException tte) {
+ sawError(tte);
+ throw tte;
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public void write(byte[] arg0) throws TTransportException {
+ try {
+ ioCount++;
+ wrappedTransport.write(arg0);
+ } catch (TTransportException tte) {
+ sawError(tte);
+ throw tte;
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public void close() {
+ try {
+ ioCount++;
+ wrappedTransport.close();
+ } finally {
+ ioCount++;
+ }
+
+ }
+
++ @Override
+ public void flush() throws TTransportException {
+ try {
+ ioCount++;
+ wrappedTransport.flush();
+ } catch (TTransportException tte) {
+ sawError(tte);
+ throw tte;
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public boolean peek() {
+ try {
+ ioCount++;
+ return wrappedTransport.peek();
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public byte[] getBuffer() {
+ try {
+ ioCount++;
+ return wrappedTransport.getBuffer();
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public int getBufferPosition() {
+ try {
+ ioCount++;
+ return wrappedTransport.getBufferPosition();
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public int getBytesRemainingInBuffer() {
+ try {
+ ioCount++;
+ return wrappedTransport.getBytesRemainingInBuffer();
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public void consumeBuffer(int len) {
+ try {
+ ioCount++;
+ wrappedTransport.consumeBuffer(len);
+ } finally {
+ ioCount++;
+ }
+ }
+
+ public ThriftTransportKey getCacheKey() {
+ return cacheKey;
+ }
+
+ }
+
+ private ThriftTransportPool() {}
+
+ public TTransport getTransport(String location, int port) throws TTransportException {
+ return getTransport(location, port, 0);
+ }
+
+ public TTransport getTransportWithDefaultTimeout(InetSocketAddress addr, AccumuloConfiguration conf) throws TTransportException {
+ return getTransport(addr.getAddress().getHostAddress(), addr.getPort(), conf.getTimeInMillis(Property.GENERAL_RPC_TIMEOUT));
+ }
+
+ public TTransport getTransport(InetSocketAddress addr, long timeout) throws TTransportException {
+ return getTransport(addr.getAddress().getHostAddress(), addr.getPort(), timeout);
+ }
+
+ public TTransport getTransportWithDefaultTimeout(String location, int port, AccumuloConfiguration conf) throws TTransportException {
+ return getTransport(location, port, conf.getTimeInMillis(Property.GENERAL_RPC_TIMEOUT));
+ }
+
+ Pair<String,TTransport> getAnyTransport(List<ThriftTransportKey> servers, boolean preferCachedConnection) throws TTransportException {
+
+ servers = new ArrayList<ThriftTransportKey>(servers);
+
+ if (preferCachedConnection) {
+ HashSet<ThriftTransportKey> serversSet = new HashSet<ThriftTransportKey>(servers);
+
+ synchronized (this) {
+
+ // randomly pick a server from the connection cache
+ serversSet.retainAll(getCache().keySet());
+
+ if (serversSet.size() > 0) {
+ ArrayList<ThriftTransportKey> cachedServers = new ArrayList<ThriftTransportKey>(serversSet);
+ Collections.shuffle(cachedServers, random);
+
+ for (ThriftTransportKey ttk : cachedServers) {
+ for (CachedConnection cachedConnection : getCache().get(ttk)) {
+ if (!cachedConnection.isReserved()) {
+ cachedConnection.setReserved(true);
+ if (log.isTraceEnabled())
+ log.trace("Using existing connection to " + ttk.getLocation() + ":" + ttk.getPort());
+ return new Pair<String,TTransport>(ttk.getLocation() + ":" + ttk.getPort(), cachedConnection.transport);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ int retryCount = 0;
+ while (servers.size() > 0 && retryCount < 10) {
+ int index = random.nextInt(servers.size());
+ ThriftTransportKey ttk = servers.get(index);
+
+ if (!preferCachedConnection) {
+ synchronized (this) {
+ List<CachedConnection> cachedConnList = getCache().get(ttk);
+ if (cachedConnList != null) {
+ for (CachedConnection cachedConnection : cachedConnList) {
+ if (!cachedConnection.isReserved()) {
+ cachedConnection.setReserved(true);
+ if (log.isTraceEnabled())
+ log.trace("Using existing connection to " + ttk.getLocation() + ":" + ttk.getPort() + " timeout " + ttk.getTimeout());
+ return new Pair<String,TTransport>(ttk.getLocation() + ":" + ttk.getPort(), cachedConnection.transport);
+ }
+ }
+ }
+ }
+ }
+
+ try {
+ return new Pair<String,TTransport>(ttk.getLocation() + ":" + ttk.getPort(), createNewTransport(ttk));
+ } catch (TTransportException tte) {
+ log.debug("Failed to connect to " + servers.get(index), tte);
+ servers.remove(index);
+ retryCount++;
+ }
+ }
+
+ throw new TTransportException("Failed to connect to a server");
+ }
+
+ public TTransport getTransport(String location, int port, long milliseconds) throws TTransportException {
+ return getTransport(new ThriftTransportKey(location, port, milliseconds));
+ }
+
+ private TTransport getTransport(ThriftTransportKey cacheKey) throws TTransportException {
+ synchronized (this) {
+ // atomically reserve location if it exist in cache
+ List<CachedConnection> ccl = getCache().get(cacheKey);
+
+ if (ccl == null) {
+ ccl = new LinkedList<CachedConnection>();
+ getCache().put(cacheKey, ccl);
+ }
+
+ for (CachedConnection cachedConnection : ccl) {
+ if (!cachedConnection.isReserved()) {
+ cachedConnection.setReserved(true);
+ if (log.isTraceEnabled())
+ log.trace("Using existing connection to " + cacheKey.getLocation() + ":" + cacheKey.getPort());
+ return cachedConnection.transport;
+ }
+ }
+ }
+
+ return createNewTransport(cacheKey);
+ }
+
+ private TTransport createNewTransport(ThriftTransportKey cacheKey) throws TTransportException {
+ TTransport transport;
+ if (cacheKey.getTimeout() == 0) {
+ transport = AddressUtil.createTSocket(cacheKey.getLocation(), cacheKey.getPort());
+ } else {
+ try {
+ transport = TTimeoutTransport.create(AddressUtil.parseAddress(cacheKey.getLocation(), cacheKey.getPort()), cacheKey.getTimeout());
+ } catch (IOException ex) {
+ throw new TTransportException(ex);
+ }
+ }
+ transport = ThriftUtil.transportFactory().getTransport(transport);
+ transport.open();
+
+ if (log.isTraceEnabled())
+ log.trace("Creating new connection to connection to " + cacheKey.getLocation() + ":" + cacheKey.getPort());
+
+ CachedTTransport tsc = new CachedTTransport(transport, cacheKey);
+
+ CachedConnection cc = new CachedConnection(tsc);
+ cc.setReserved(true);
+
+ try {
+ synchronized (this) {
+ List<CachedConnection> ccl = getCache().get(cacheKey);
+
+ if (ccl == null) {
+ ccl = new LinkedList<CachedConnection>();
+ getCache().put(cacheKey, ccl);
+ }
+
+ ccl.add(cc);
+ }
+ } catch (TransportPoolShutdownException e) {
+ cc.transport.close();
+ throw e;
+ }
+ return cc.transport;
+ }
+
+ public void returnTransport(TTransport tsc) {
+ if (tsc == null) {
+ return;
+ }
+
+ boolean existInCache = false;
+ CachedTTransport ctsc = (CachedTTransport) tsc;
+
+ ArrayList<CachedConnection> closeList = new ArrayList<ThriftTransportPool.CachedConnection>();
+
+ synchronized (this) {
+ List<CachedConnection> ccl = getCache().get(ctsc.getCacheKey());
+ for (Iterator<CachedConnection> iterator = ccl.iterator(); iterator.hasNext();) {
+ CachedConnection cachedConnection = iterator.next();
+ if (cachedConnection.transport == tsc) {
+ if (ctsc.sawError) {
+ closeList.add(cachedConnection);
+ iterator.remove();
+
+ if (log.isTraceEnabled())
+ log.trace("Returned connection had error " + ctsc.getCacheKey());
+
+ Long ecount = errorCount.get(ctsc.getCacheKey());
+ if (ecount == null)
+ ecount = 0l;
+ ecount++;
+ errorCount.put(ctsc.getCacheKey(), ecount);
+
+ Long etime = errorTime.get(ctsc.getCacheKey());
+ if (etime == null) {
+ errorTime.put(ctsc.getCacheKey(), System.currentTimeMillis());
+ }
+
+ if (ecount >= ERROR_THRESHOLD && !serversWarnedAbout.contains(ctsc.getCacheKey())) {
+ log.warn("Server " + ctsc.getCacheKey() + " had " + ecount + " failures in a short time period, will not complain anymore ");
+ serversWarnedAbout.add(ctsc.getCacheKey());
+ }
+
+ cachedConnection.setReserved(false);
+
+ } else {
+
+ if (log.isTraceEnabled())
+ log.trace("Returned connection " + ctsc.getCacheKey() + " ioCount : " + cachedConnection.transport.ioCount);
+
+ cachedConnection.lastReturnTime = System.currentTimeMillis();
+ cachedConnection.setReserved(false);
+ }
+ existInCache = true;
+ break;
+ }
+ }
+
+ // remove all unreserved cached connection when a sever has an error, not just the connection that was returned
+ if (ctsc.sawError) {
+ for (Iterator<CachedConnection> iterator = ccl.iterator(); iterator.hasNext();) {
+ CachedConnection cachedConnection = iterator.next();
+ if (!cachedConnection.isReserved()) {
+ closeList.add(cachedConnection);
+ iterator.remove();
+ }
+ }
+ }
+ }
+
+ // close outside of sync block
+ for (CachedConnection cachedConnection : closeList) {
+ try {
+ cachedConnection.transport.close();
+ } catch (Exception e) {
+ log.debug("Failed to close connection w/ errors", e);
+ }
+ }
+
+ if (!existInCache) {
+ log.warn("Returned tablet server connection to cache that did not come from cache");
+ // close outside of sync block
+ tsc.close();
+ }
+ }
+
+ /**
+ * Set the time after which idle connections should be closed
- *
- * @param time
+ */
+ public synchronized void setIdleTime(long time) {
+ this.killTime = time;
+ log.debug("Set thrift transport pool idle time to " + time);
+ }
+
+ private static ThriftTransportPool instance = new ThriftTransportPool();
+ private static final AtomicBoolean daemonStarted = new AtomicBoolean(false);
+
+ public static ThriftTransportPool getInstance() {
+ SecurityManager sm = System.getSecurityManager();
+ if (sm != null) {
+ sm.checkPermission(TRANSPORT_POOL_PERMISSION);
+ }
+
+ if (daemonStarted.compareAndSet(false, true)) {
+ CountDownLatch closerExitLatch = new CountDownLatch(1);
+ new Daemon(new Closer(instance, closerExitLatch), "Thrift Connection Pool Checker").start();
+ instance.setCloserExitLatch(closerExitLatch);
+ }
+ return instance;
+ }
+
+ private synchronized void setCloserExitLatch(CountDownLatch closerExitLatch) {
+ this.closerExitLatch = closerExitLatch;
+ }
+
+ public void shutdown() {
+ synchronized (this) {
+ if (cache == null)
+ return;
+
+ // close any connections in the pool... even ones that are in use
+ for (List<CachedConnection> ccl : getCache().values()) {
+ Iterator<CachedConnection> iter = ccl.iterator();
+ while (iter.hasNext()) {
+ CachedConnection cc = iter.next();
+ try {
+ cc.transport.close();
+ } catch (Exception e) {
+ log.debug("Error closing transport during shutdown", e);
+ }
+ }
+ }
+
+ // this will render the pool unusable and cause the background thread to exit
+ this.cache = null;
+ }
+
+ try {
+ closerExitLatch.await();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private Map<ThriftTransportKey,List<CachedConnection>> getCache() {
+ if (cache == null)
+ throw new TransportPoolShutdownException();
+ return cache;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
index ee4aca5,0000000..d7be37c
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
@@@ -1,511 -1,0 +1,510 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapred;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.mapreduce.lib.util.OutputConfigurator;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.SecurityErrorCode;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.OutputFormat;
+import org.apache.hadoop.mapred.RecordWriter;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.util.Progressable;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * This class allows MapReduce jobs to use Accumulo as the sink for data. This {@link OutputFormat} accepts keys and values of type {@link Text} (for a table
+ * name) and {@link Mutation} from the Map and Reduce functions.
+ *
+ * The user must specify the following via static configurator methods:
+ *
+ * <ul>
+ * <li>{@link AccumuloOutputFormat#setConnectorInfo(JobConf, String, AuthenticationToken)}
+ * <li>{@link AccumuloOutputFormat#setZooKeeperInstance(JobConf, String, String)} OR {@link AccumuloOutputFormat#setMockInstance(JobConf, String)}
+ * </ul>
+ *
+ * Other static methods are optional.
+ */
+public class AccumuloOutputFormat implements OutputFormat<Text,Mutation> {
+
+ private static final Class<?> CLASS = AccumuloOutputFormat.class;
+ protected static final Logger log = Logger.getLogger(CLASS);
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
+ * conversion to a string, and is not intended to be secure.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission if {@link #setCreateTables(JobConf, boolean)} is set to true)
+ * @param token
+ * the user's password
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public static void setConnectorInfo(JobConf job, String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ OutputConfigurator.setConnectorInfo(CLASS, job, principal, token);
+ }
+
+ /**
+ * Determines if the connector has been configured.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the connector has been configured, false otherwise
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static Boolean isConnectorInfoSet(JobConf job) {
+ return OutputConfigurator.isConnectorInfoSet(CLASS, job);
+ }
+
+ /**
+ * Gets the principal from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static String getPrincipal(JobConf job) {
+ return OutputConfigurator.getPrincipal(CLASS, job);
+ }
+
+ /**
+ * Gets the serialized token class from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static String getTokenClass(JobConf job) {
+ return OutputConfigurator.getTokenClass(CLASS, job);
+ }
+
+ /**
+ * Gets the password from the configuration. WARNING: The password is stored in the Configuration and shared with all MapReduce tasks; It is BASE64 encoded to
+ * provide a charset safe conversion to a string, and is not intended to be secure.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the decoded user password
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static byte[] getToken(JobConf job) {
+ return OutputConfigurator.getToken(CLASS, job);
+ }
+
+ /**
+ * Configures a {@link ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @param zooKeepers
+ * a comma-separated list of zookeeper servers
+ * @since 1.5.0
+ */
+ public static void setZooKeeperInstance(JobConf job, String instanceName, String zooKeepers) {
+ OutputConfigurator.setZooKeeperInstance(CLASS, job, instanceName, zooKeepers);
+ }
+
+ /**
+ * Configures a {@link MockInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @since 1.5.0
+ */
+ public static void setMockInstance(JobConf job, String instanceName) {
+ OutputConfigurator.setMockInstance(CLASS, job, instanceName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link Instance} based on the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return an Accumulo instance
+ * @since 1.5.0
+ * @see #setZooKeeperInstance(JobConf, String, String)
+ * @see #setMockInstance(JobConf, String)
+ */
+ protected static Instance getInstance(JobConf job) {
+ return OutputConfigurator.getInstance(CLASS, job);
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param level
+ * the logging level
+ * @since 1.5.0
+ */
+ public static void setLogLevel(JobConf job, Level level) {
+ OutputConfigurator.setLogLevel(CLASS, job, level);
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the log level
+ * @since 1.5.0
+ * @see #setLogLevel(JobConf, Level)
+ */
+ protected static Level getLogLevel(JobConf job) {
+ return OutputConfigurator.getLogLevel(CLASS, job);
+ }
+
+ /**
+ * Sets the default table name to use if one emits a null in place of a table name for a given mutation. Table names can only be alpha-numeric and
+ * underscores.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.5.0
+ */
+ public static void setDefaultTableName(JobConf job, String tableName) {
+ OutputConfigurator.setDefaultTableName(CLASS, job, tableName);
+ }
+
+ /**
+ * Gets the default table name from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the default table name
+ * @since 1.5.0
+ * @see #setDefaultTableName(JobConf, String)
+ */
+ protected static String getDefaultTableName(JobConf job) {
+ return OutputConfigurator.getDefaultTableName(CLASS, job);
+ }
+
+ /**
+ * Sets the configuration for for the job's {@link BatchWriter} instances. If not set, a new {@link BatchWriterConfig}, with sensible built-in defaults is
+ * used. Setting the configuration multiple times overwrites any previous configuration.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param bwConfig
+ * the configuration for the {@link BatchWriter}
+ * @since 1.5.0
+ */
+ public static void setBatchWriterOptions(JobConf job, BatchWriterConfig bwConfig) {
+ OutputConfigurator.setBatchWriterOptions(CLASS, job, bwConfig);
+ }
+
+ /**
+ * Gets the {@link BatchWriterConfig} settings.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the configuration object
+ * @since 1.5.0
+ * @see #setBatchWriterOptions(JobConf, BatchWriterConfig)
+ */
+ protected static BatchWriterConfig getBatchWriterOptions(JobConf job) {
+ return OutputConfigurator.getBatchWriterOptions(CLASS, job);
+ }
+
+ /**
+ * Sets the directive to create new tables, as necessary. Table names can only be alpha-numeric and underscores.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setCreateTables(JobConf job, boolean enableFeature) {
+ OutputConfigurator.setCreateTables(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether tables are permitted to be created as needed.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the feature is disabled, false otherwise
+ * @since 1.5.0
+ * @see #setCreateTables(JobConf, boolean)
+ */
+ protected static Boolean canCreateTables(JobConf job) {
+ return OutputConfigurator.canCreateTables(CLASS, job);
+ }
+
+ /**
+ * Sets the directive to use simulation mode for this job. In simulation mode, no output is produced. This is useful for testing.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setSimulationMode(JobConf job, boolean enableFeature) {
+ OutputConfigurator.setSimulationMode(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether this feature is enabled.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setSimulationMode(JobConf, boolean)
+ */
+ protected static Boolean getSimulationMode(JobConf job) {
+ return OutputConfigurator.getSimulationMode(CLASS, job);
+ }
+
+ /**
+ * A base class to be used to create {@link RecordWriter} instances that write to Accumulo.
+ */
+ protected static class AccumuloRecordWriter implements RecordWriter<Text,Mutation> {
+ private MultiTableBatchWriter mtbw = null;
+ private HashMap<Text,BatchWriter> bws = null;
+ private Text defaultTableName = null;
+
+ private boolean simulate = false;
+ private boolean createTables = false;
+
+ private long mutCount = 0;
+ private long valCount = 0;
+
+ private Connector conn;
+
+ protected AccumuloRecordWriter(JobConf job) throws AccumuloException, AccumuloSecurityException, IOException {
+ Level l = getLogLevel(job);
+ if (l != null)
+ log.setLevel(getLogLevel(job));
+ this.simulate = getSimulationMode(job);
+ this.createTables = canCreateTables(job);
+
+ if (simulate)
+ log.info("Simulating output only. No writes to tables will occur");
+
+ this.bws = new HashMap<Text,BatchWriter>();
+
+ String tname = getDefaultTableName(job);
+ this.defaultTableName = (tname == null) ? null : new Text(tname);
+
+ if (!simulate) {
+ this.conn = getInstance(job).getConnector(getPrincipal(job), CredentialHelper.extractToken(getTokenClass(job), getToken(job)));
+ mtbw = conn.createMultiTableBatchWriter(getBatchWriterOptions(job));
+ }
+ }
+
+ /**
+ * Push a mutation into a table. If table is null, the defaultTable will be used. If canCreateTable is set, the table will be created if it does not exist.
+ * The table name must only contain alphanumerics and underscore.
+ */
+ @Override
+ public void write(Text table, Mutation mutation) throws IOException {
+ if (table == null || table.toString().isEmpty())
+ table = this.defaultTableName;
+
+ if (!simulate && table == null)
+ throw new IOException("No table or default table specified. Try simulation mode next time");
+
+ ++mutCount;
+ valCount += mutation.size();
+ printMutation(table, mutation);
+
+ if (simulate)
+ return;
+
+ if (!bws.containsKey(table))
+ try {
+ addTable(table);
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new IOException(e);
+ }
+
+ try {
+ bws.get(table).addMutation(mutation);
+ } catch (MutationsRejectedException e) {
+ throw new IOException(e);
+ }
+ }
+
+ public void addTable(Text tableName) throws AccumuloException, AccumuloSecurityException {
+ if (simulate) {
+ log.info("Simulating adding table: " + tableName);
+ return;
+ }
+
+ log.debug("Adding table: " + tableName);
+ BatchWriter bw = null;
+ String table = tableName.toString();
+
+ if (createTables && !conn.tableOperations().exists(table)) {
+ try {
+ conn.tableOperations().create(table);
+ } catch (AccumuloSecurityException e) {
+ log.error("Accumulo security violation creating " + table, e);
+ throw e;
+ } catch (TableExistsException e) {
+ // Shouldn't happen
+ }
+ }
+
+ try {
+ bw = mtbw.getBatchWriter(table);
+ } catch (TableNotFoundException e) {
+ log.error("Accumulo table " + table + " doesn't exist and cannot be created.", e);
+ throw new AccumuloException(e);
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (AccumuloSecurityException e) {
+ throw e;
+ }
+
+ if (bw != null)
+ bws.put(tableName, bw);
+ }
+
+ private int printMutation(Text table, Mutation m) {
+ if (log.isTraceEnabled()) {
+ log.trace(String.format("Table %s row key: %s", table, hexDump(m.getRow())));
+ for (ColumnUpdate cu : m.getUpdates()) {
+ log.trace(String.format("Table %s column: %s:%s", table, hexDump(cu.getColumnFamily()), hexDump(cu.getColumnQualifier())));
+ log.trace(String.format("Table %s security: %s", table, new ColumnVisibility(cu.getColumnVisibility()).toString()));
+ log.trace(String.format("Table %s value: %s", table, hexDump(cu.getValue())));
+ }
+ }
+ return m.getUpdates().size();
+ }
+
+ private String hexDump(byte[] ba) {
+ StringBuilder sb = new StringBuilder();
+ for (byte b : ba) {
+ if ((b > 0x20) && (b < 0x7e))
+ sb.append((char) b);
+ else
+ sb.append(String.format("x%02x", b));
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public void close(Reporter reporter) throws IOException {
+ log.debug("mutations written: " + mutCount + ", values written: " + valCount);
+ if (simulate)
+ return;
+
+ try {
+ mtbw.close();
+ } catch (MutationsRejectedException e) {
+ if (e.getAuthorizationFailuresMap().size() >= 0) {
+ HashMap<String,Set<SecurityErrorCode>> tables = new HashMap<String,Set<SecurityErrorCode>>();
+ for (Entry<KeyExtent,Set<SecurityErrorCode>> ke : e.getAuthorizationFailuresMap().entrySet()) {
+ Set<SecurityErrorCode> secCodes = tables.get(ke.getKey().getTableId().toString());
+ if (secCodes == null) {
+ secCodes = new HashSet<SecurityErrorCode>();
+ tables.put(ke.getKey().getTableId().toString(), secCodes);
+ }
+ secCodes.addAll(ke.getValue());
+ }
+
+ log.error("Not authorized to write to tables : " + tables);
+ }
+
+ if (e.getConstraintViolationSummaries().size() > 0) {
+ log.error("Constraint violations : " + e.getConstraintViolationSummaries().size());
+ }
+ }
+ }
+ }
+
+ @Override
+ public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException {
+ if (!isConnectorInfoSet(job))
+ throw new IOException("Connector info has not been set.");
+ try {
+ // if the instance isn't configured, it will complain here
+ Connector c = getInstance(job).getConnector(getPrincipal(job), CredentialHelper.extractToken(getTokenClass(job), getToken(job)));
+ if (!c.securityOperations().authenticateUser(getPrincipal(job), CredentialHelper.extractToken(getTokenClass(job), getToken(job))))
+ throw new IOException("Unable to authenticate user");
+ } catch (AccumuloException e) {
+ throw new IOException(e);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public RecordWriter<Text,Mutation> getRecordWriter(FileSystem ignored, JobConf job, String name, Progressable progress) throws IOException {
+ try {
+ return new AccumuloRecordWriter(job);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
index 16efa89,0000000..bc568e8
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
@@@ -1,925 -1,0 +1,924 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapred;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientSideIteratorScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.RowIterator;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableDeletedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.impl.OfflineScanner;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.TabletLocator;
+import org.apache.accumulo.core.client.mapreduce.lib.util.InputConfigurator;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * This abstract {@link InputFormat} class allows MapReduce jobs to use Accumulo as the source of K,V pairs.
+ * <p>
+ * Subclasses must implement a {@link #getRecordReader(InputSplit, JobConf, Reporter)} to provide a {@link RecordReader} for K,V.
+ * <p>
+ * A static base class, RecordReaderBase, is provided to retrieve Accumulo {@link Key}/{@link Value} pairs, but one must implement its
+ * {@link RecordReaderBase#next(Object, Object)} to transform them to the desired generic types K,V.
+ * <p>
+ * See {@link AccumuloInputFormat} for an example implementation.
+ */
+public abstract class InputFormatBase<K,V> implements InputFormat<K,V> {
+
+ private static final Class<?> CLASS = AccumuloInputFormat.class;
+ protected static final Logger log = Logger.getLogger(CLASS);
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
+ * conversion to a string, and is not intended to be secure.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission)
+ * @param token
+ * the user's password
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public static void setConnectorInfo(JobConf job, String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ InputConfigurator.setConnectorInfo(CLASS, job, principal, token);
+ }
+
+ /**
+ * Determines if the connector has been configured.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the connector has been configured, false otherwise
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static Boolean isConnectorInfoSet(JobConf job) {
+ return InputConfigurator.isConnectorInfoSet(CLASS, job);
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static String getPrincipal(JobConf job) {
+ return InputConfigurator.getPrincipal(CLASS, job);
+ }
+
+ /**
+ * Gets the serialized token class from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static String getTokenClass(JobConf job) {
+ return InputConfigurator.getTokenClass(CLASS, job);
+ }
+
+ /**
+ * Gets the password from the configuration. WARNING: The password is stored in the Configuration and shared with all MapReduce tasks; It is BASE64 encoded to
+ * provide a charset safe conversion to a string, and is not intended to be secure.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the decoded user password
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static byte[] getToken(JobConf job) {
+ return InputConfigurator.getToken(CLASS, job);
+ }
+
+ /**
+ * Configures a {@link ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @param zooKeepers
+ * a comma-separated list of zookeeper servers
+ * @since 1.5.0
+ */
+ public static void setZooKeeperInstance(JobConf job, String instanceName, String zooKeepers) {
+ InputConfigurator.setZooKeeperInstance(CLASS, job, instanceName, zooKeepers);
+ }
+
+ /**
+ * Configures a {@link MockInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @since 1.5.0
+ */
+ public static void setMockInstance(JobConf job, String instanceName) {
+ InputConfigurator.setMockInstance(CLASS, job, instanceName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link Instance} based on the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return an Accumulo instance
+ * @since 1.5.0
+ * @see #setZooKeeperInstance(JobConf, String, String)
+ * @see #setMockInstance(JobConf, String)
+ */
+ protected static Instance getInstance(JobConf job) {
+ return InputConfigurator.getInstance(CLASS, job);
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param level
+ * the logging level
+ * @since 1.5.0
+ */
+ public static void setLogLevel(JobConf job, Level level) {
+ InputConfigurator.setLogLevel(CLASS, job, level);
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the log level
+ * @since 1.5.0
+ * @see #setLogLevel(JobConf, Level)
+ */
+ protected static Level getLogLevel(JobConf job) {
+ return InputConfigurator.getLogLevel(CLASS, job);
+ }
+
+ /**
+ * Sets the name of the input table, over which this job will scan.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.5.0
+ */
+ public static void setInputTableName(JobConf job, String tableName) {
+ InputConfigurator.setInputTableName(CLASS, job, tableName);
+ }
+
+ /**
+ * Gets the table name from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the table name
+ * @since 1.5.0
+ * @see #setInputTableName(JobConf, String)
+ */
+ protected static String getInputTableName(JobConf job) {
+ return InputConfigurator.getInputTableName(CLASS, job);
+ }
+
+ /**
+ * Sets the {@link Authorizations} used to scan. Must be a subset of the user's authorization. Defaults to the empty set.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param auths
+ * the user's authorizations
+ * @since 1.5.0
+ */
+ public static void setScanAuthorizations(JobConf job, Authorizations auths) {
+ InputConfigurator.setScanAuthorizations(CLASS, job, auths);
+ }
+
+ /**
+ * Gets the authorizations to set for the scans from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the Accumulo scan authorizations
+ * @since 1.5.0
+ * @see #setScanAuthorizations(JobConf, Authorizations)
+ */
+ protected static Authorizations getScanAuthorizations(JobConf job) {
+ return InputConfigurator.getScanAuthorizations(CLASS, job);
+ }
+
+ /**
+ * Sets the input ranges to scan for this job. If not set, the entire table will be scanned.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param ranges
+ * the ranges that will be mapped over
+ * @since 1.5.0
+ */
+ public static void setRanges(JobConf job, Collection<Range> ranges) {
+ InputConfigurator.setRanges(CLASS, job, ranges);
+ }
+
+ /**
+ * Gets the ranges to scan over from a job.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the ranges
+ * @throws IOException
+ * if the ranges have been encoded improperly
+ * @since 1.5.0
+ * @see #setRanges(JobConf, Collection)
+ */
+ protected static List<Range> getRanges(JobConf job) throws IOException {
+ return InputConfigurator.getRanges(CLASS, job);
+ }
+
+ /**
+ * Restricts the columns that will be mapped over for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param columnFamilyColumnQualifierPairs
+ * a pair of {@link Text} objects corresponding to column family and column qualifier. If the column qualifier is null, the entire column family is
+ * selected. An empty set is the default and is equivalent to scanning the all columns.
+ * @since 1.5.0
+ */
+ public static void fetchColumns(JobConf job, Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
+ InputConfigurator.fetchColumns(CLASS, job, columnFamilyColumnQualifierPairs);
+ }
+
+ /**
+ * Gets the columns to be mapped over from this job.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return a set of columns
+ * @since 1.5.0
+ * @see #fetchColumns(JobConf, Collection)
+ */
+ protected static Set<Pair<Text,Text>> getFetchedColumns(JobConf job) {
+ return InputConfigurator.getFetchedColumns(CLASS, job);
+ }
+
+ /**
+ * Encode an iterator on the input for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param cfg
+ * the configuration of the iterator
+ * @since 1.5.0
+ */
+ public static void addIterator(JobConf job, IteratorSetting cfg) {
+ InputConfigurator.addIterator(CLASS, job, cfg);
+ }
+
+ /**
+ * Gets a list of the iterator settings (for iterators to apply to a scanner) from this configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return a list of iterators
+ * @since 1.5.0
+ * @see #addIterator(JobConf, IteratorSetting)
+ */
+ protected static List<IteratorSetting> getIterators(JobConf job) {
+ return InputConfigurator.getIterators(CLASS, job);
+ }
+
+ /**
+ * Controls the automatic adjustment of ranges for this job. This feature merges overlapping ranges, then splits them to align with tablet boundaries.
+ * Disabling this feature will cause exactly one Map task to be created for each specified range. The default setting is enabled. *
+ *
+ * <p>
+ * By default, this feature is <b>enabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @see #setRanges(JobConf, Collection)
+ * @since 1.5.0
+ */
+ public static void setAutoAdjustRanges(JobConf job, boolean enableFeature) {
+ InputConfigurator.setAutoAdjustRanges(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has auto-adjust ranges enabled.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return false if the feature is disabled, true otherwise
+ * @since 1.5.0
+ * @see #setAutoAdjustRanges(JobConf, boolean)
+ */
+ protected static boolean getAutoAdjustRanges(JobConf job) {
+ return InputConfigurator.getAutoAdjustRanges(CLASS, job);
+ }
+
+ /**
+ * Controls the use of the {@link IsolatedScanner} in this job.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setScanIsolation(JobConf job, boolean enableFeature) {
+ InputConfigurator.setScanIsolation(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has isolation enabled.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setScanIsolation(JobConf, boolean)
+ */
+ protected static boolean isIsolated(JobConf job) {
+ return InputConfigurator.isIsolated(CLASS, job);
+ }
+
+ /**
+ * Controls the use of the {@link ClientSideIteratorScanner} in this job. Enabling this feature will cause the iterator stack to be constructed within the Map
+ * task, rather than within the Accumulo TServer. To use this feature, all classes needed for those iterators must be available on the classpath for the task.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setLocalIterators(JobConf job, boolean enableFeature) {
+ InputConfigurator.setLocalIterators(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration uses local iterators.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setLocalIterators(JobConf, boolean)
+ */
+ protected static boolean usesLocalIterators(JobConf job) {
+ return InputConfigurator.usesLocalIterators(CLASS, job);
+ }
+
+ /**
+ * <p>
+ * Enable reading offline tables. By default, this feature is disabled and only online tables are scanned. This will make the map reduce job directly read the
+ * table's files. If the table is not offline, then the job will fail. If the table comes online during the map reduce job, it is likely that the job will
+ * fail.
+ *
+ * <p>
+ * To use this option, the map reduce user will need access to read the Accumulo directory in HDFS.
+ *
+ * <p>
+ * Reading the offline table will create the scan time iterator stack in the map process. So any iterators that are configured for the table will need to be
+ * on the mapper's classpath.
+ *
+ * <p>
+ * One way to use this feature is to clone a table, take the clone offline, and use the clone as the input table for a map reduce job. If you plan to map
+ * reduce over the data many times, it may be better to the compact the table, clone it, take it offline, and use the clone for all map reduce jobs. The
+ * reason to do this is that compaction will reduce each tablet in the table to one file, and it is faster to read from one file.
+ *
+ * <p>
+ * There are two possible advantages to reading a tables file directly out of HDFS. First, you may see better read performance. Second, it will support
+ * speculative execution better. When reading an online table speculative execution can put more load on an already slow tablet server.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setOfflineTableScan(JobConf job, boolean enableFeature) {
+ InputConfigurator.setOfflineTableScan(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has the offline table scan feature enabled.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setOfflineTableScan(JobConf, boolean)
+ */
+ protected static boolean isOfflineScan(JobConf job) {
+ return InputConfigurator.isOfflineScan(CLASS, job);
+ }
+
+ /**
+ * Initializes an Accumulo {@link TabletLocator} based on the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return an Accumulo tablet locator
+ * @throws TableNotFoundException
+ * if the table name set on the configuration doesn't exist
+ * @since 1.5.0
+ */
+ protected static TabletLocator getTabletLocator(JobConf job) throws TableNotFoundException {
+ return InputConfigurator.getTabletLocator(CLASS, job);
+ }
+
+ // InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext job)
+ /**
+ * Check whether a configuration is fully configured to be used with an Accumulo {@link org.apache.hadoop.mapreduce.InputFormat}.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @throws IOException
+ * if the context is improperly configured
+ * @since 1.5.0
+ */
+ protected static void validateOptions(JobConf job) throws IOException {
+ InputConfigurator.validateOptions(CLASS, job);
+ }
+
+ /**
+ * An abstract base class to be used to create {@link RecordReader} instances that convert from Accumulo {@link Key}/{@link Value} pairs to the user's K/V
+ * types.
+ *
+ * Subclasses must implement {@link #next(Object, Object)} to update key and value, and also to update the following variables:
+ * <ul>
+ * <li>Key {@link #currentKey} (used for progress reporting)</li>
+ * <li>int {@link #numKeysRead} (used for progress reporting)</li>
+ * </ul>
+ */
+ protected abstract static class RecordReaderBase<K,V> implements RecordReader<K,V> {
+ protected long numKeysRead;
+ protected Iterator<Entry<Key,Value>> scannerIterator;
+ protected org.apache.accumulo.core.client.mapred.RangeInputSplit split;
+
+ /**
+ * Apply the configured iterators from the configuration to the scanner.
+ *
+ * @param scanner
+ * the scanner to configure
+ */
+ protected void setupIterators(List<IteratorSetting> iterators, Scanner scanner) {
+ for (IteratorSetting iterator : iterators) {
+ scanner.addScanIterator(iterator);
+ }
+ }
+
+ /**
+ * Initialize a scanner over the given input split using this task attempt configuration.
+ */
+ public void initialize(InputSplit inSplit, JobConf job) throws IOException {
+ Scanner scanner;
+ split = (org.apache.accumulo.core.client.mapred.RangeInputSplit) inSplit;
+ log.debug("Initializing input split: " + split.getRange());
+
+ Instance instance = split.getInstance();
+ if (null == instance) {
+ instance = getInstance(job);
+ }
+
+ String principal = split.getPrincipal();
+ if (null == principal) {
+ principal = getPrincipal(job);
+ }
+
+ AuthenticationToken token = split.getToken();
+ if (null == token) {
+ String tokenClass = getTokenClass(job);
+ byte[] tokenBytes = getToken(job);
+ try {
+ token = CredentialHelper.extractToken(tokenClass, tokenBytes);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+ }
+
+ Authorizations authorizations = split.getAuths();
+ if (null == authorizations) {
+ authorizations = getScanAuthorizations(job);
+ }
+
+ String table = split.getTable();
+ if (null == table) {
+ table = getInputTableName(job);
+ }
+
+ Boolean isOffline = split.isOffline();
+ if (null == isOffline) {
+ isOffline = isOfflineScan(job);
+ }
+
+ Boolean isIsolated = split.isIsolatedScan();
+ if (null == isIsolated) {
+ isIsolated = isIsolated(job);
+ }
+
+ Boolean usesLocalIterators = split.usesLocalIterators();
+ if (null == usesLocalIterators) {
+ usesLocalIterators = usesLocalIterators(job);
+ }
+
+ List<IteratorSetting> iterators = split.getIterators();
+ if (null == iterators) {
+ iterators = getIterators(job);
+ }
+
+ Set<Pair<Text,Text>> columns = split.getFetchedColumns();
+ if (null == columns) {
+ columns = getFetchedColumns(job);
+ }
+
+ try {
+ log.debug("Creating connector with user: " + principal);
+ Connector conn = instance.getConnector(principal, token);
+ log.debug("Creating scanner for table: " + table);
+ log.debug("Authorizations are: " + authorizations);
+ if (isOffline) {
+ String tokenClass = token.getClass().getCanonicalName();
+ ByteBuffer tokenBuffer = ByteBuffer.wrap(CredentialHelper.toBytes(token));
+ scanner = new OfflineScanner(instance, new TCredentials(principal, tokenClass, tokenBuffer, instance.getInstanceID()), Tables.getTableId(instance,
+ table), authorizations);
+ } else {
+ scanner = conn.createScanner(table, authorizations);
+ }
+ if (isIsolated) {
+ log.info("Creating isolated scanner");
+ scanner = new IsolatedScanner(scanner);
+ }
+ if (usesLocalIterators) {
+ log.info("Using local iterators");
+ scanner = new ClientSideIteratorScanner(scanner);
+ }
+ setupIterators(iterators, scanner);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ // setup a scanner within the bounds of this split
+ for (Pair<Text,Text> c : columns) {
+ if (c.getSecond() != null) {
+ log.debug("Fetching column " + c.getFirst() + ":" + c.getSecond());
+ scanner.fetchColumn(c.getFirst(), c.getSecond());
+ } else {
+ log.debug("Fetching column family " + c.getFirst());
+ scanner.fetchColumnFamily(c.getFirst());
+ }
+ }
+
+ scanner.setRange(split.getRange());
+
+ numKeysRead = 0;
+
+ // do this last after setting all scanner options
+ scannerIterator = scanner.iterator();
+ }
+
+ @Override
+ public void close() {}
+
+ @Override
+ public long getPos() throws IOException {
+ return numKeysRead;
+ }
+
+ @Override
+ public float getProgress() throws IOException {
+ if (numKeysRead > 0 && currentKey == null)
+ return 1.0f;
+ return split.getProgress(currentKey);
+ }
+
+ protected Key currentKey = null;
+
+ }
+
+ Map<String,Map<KeyExtent,List<Range>>> binOfflineTable(JobConf job, String tableName, List<Range> ranges) throws TableNotFoundException, AccumuloException,
+ AccumuloSecurityException {
+
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+
+ Instance instance = getInstance(job);
+ Connector conn = instance.getConnector(getPrincipal(job), CredentialHelper.extractToken(getTokenClass(job), getToken(job)));
+ String tableId = Tables.getTableId(instance, tableName);
+
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ Tables.clearCache(instance);
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ throw new AccumuloException("Table is online " + tableName + "(" + tableId + ") cannot scan table in offline mode ");
+ }
+ }
+
+ for (Range range : ranges) {
+ Text startRow;
+
+ if (range.getStartKey() != null)
+ startRow = range.getStartKey().getRow();
+ else
+ startRow = new Text();
+
+ Range metadataRange = new Range(new KeyExtent(new Text(tableId), startRow, null).getMetadataEntry(), true, null, false);
+ Scanner scanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ Constants.METADATA_PREV_ROW_COLUMN.fetch(scanner);
+ scanner.fetchColumnFamily(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY);
+ scanner.setRange(metadataRange);
+
+ RowIterator rowIter = new RowIterator(scanner);
+
+ KeyExtent lastExtent = null;
+
+ while (rowIter.hasNext()) {
+ Iterator<Entry<Key,Value>> row = rowIter.next();
+ String last = "";
+ KeyExtent extent = null;
+ String location = null;
+
+ while (row.hasNext()) {
+ Entry<Key,Value> entry = row.next();
+ Key key = entry.getKey();
+
+ if (key.getColumnFamily().equals(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY)) {
+ last = entry.getValue().toString();
+ }
+
+ if (key.getColumnFamily().equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY)
+ || key.getColumnFamily().equals(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY)) {
+ location = entry.getValue().toString();
+ }
+
+ if (Constants.METADATA_PREV_ROW_COLUMN.hasColumns(key)) {
+ extent = new KeyExtent(key.getRow(), entry.getValue());
+ }
+
+ }
+
+ if (location != null)
+ return null;
+
+ if (!extent.getTableId().toString().equals(tableId)) {
+ throw new AccumuloException("Saw unexpected table Id " + tableId + " " + extent);
+ }
+
+ if (lastExtent != null && !extent.isPreviousExtent(lastExtent)) {
+ throw new AccumuloException(" " + lastExtent + " is not previous extent " + extent);
+ }
+
+ Map<KeyExtent,List<Range>> tabletRanges = binnedRanges.get(last);
+ if (tabletRanges == null) {
+ tabletRanges = new HashMap<KeyExtent,List<Range>>();
+ binnedRanges.put(last, tabletRanges);
+ }
+
+ List<Range> rangeList = tabletRanges.get(extent);
+ if (rangeList == null) {
+ rangeList = new ArrayList<Range>();
+ tabletRanges.put(extent, rangeList);
+ }
+
+ rangeList.add(range);
+
+ if (extent.getEndRow() == null || range.afterEndKey(new Key(extent.getEndRow()).followingKey(PartialKey.ROW))) {
+ break;
+ }
+
+ lastExtent = extent;
+ }
+
+ }
+
+ return binnedRanges;
+ }
+
+ /**
+ * Read the metadata table to get tablets and match up ranges to them.
+ */
+ @Override
+ public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
+ Level logLevel = getLogLevel(job);
+ log.setLevel(logLevel);
+
+ validateOptions(job);
+
+ String tableName = getInputTableName(job);
+ boolean autoAdjust = getAutoAdjustRanges(job);
+ List<Range> ranges = autoAdjust ? Range.mergeOverlapping(getRanges(job)) : getRanges(job);
+ Instance instance = getInstance(job);
+ boolean offline = isOfflineScan(job);
+ boolean isolated = isIsolated(job);
+ boolean localIterators = usesLocalIterators(job);
+ boolean mockInstance = (null != instance && MockInstance.class.equals(instance.getClass()));
+ Set<Pair<Text,Text>> fetchedColumns = getFetchedColumns(job);
+ Authorizations auths = getScanAuthorizations(job);
+ String principal = getPrincipal(job);
+ String tokenClass = getTokenClass(job);
+ byte[] tokenBytes = getToken(job);
+
+ AuthenticationToken token;
+ try {
+ token = CredentialHelper.extractToken(tokenClass, tokenBytes);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+
+ List<IteratorSetting> iterators = getIterators(job);
+
+ if (ranges.isEmpty()) {
+ ranges = new ArrayList<Range>(1);
+ ranges.add(new Range());
+ }
+
+ // get the metadata information for these ranges
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+ TabletLocator tl;
+ try {
+ if (isOfflineScan(job)) {
+ binnedRanges = binOfflineTable(job, tableName, ranges);
+ while (binnedRanges == null) {
+ // Some tablets were still online, try again
+ UtilWaitThread.sleep(100 + (int) (Math.random() * 100)); // sleep randomly between 100 and 200 ms
+ binnedRanges = binOfflineTable(job, tableName, ranges);
+ }
+ } else {
+ String tableId = null;
+ tl = getTabletLocator(job);
+ // its possible that the cache could contain complete, but old information about a tables tablets... so clear it
+ tl.invalidateCache();
+ while (!tl.binRanges(ranges, binnedRanges, new TCredentials(principal, tokenClass, ByteBuffer.wrap(tokenBytes), instance.getInstanceID())).isEmpty()) {
+ if (!(instance instanceof MockInstance)) {
+ if (tableId == null)
+ tableId = Tables.getTableId(instance, tableName);
+ if (!Tables.exists(instance, tableId))
+ throw new TableDeletedException(tableId);
+ if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
+ throw new TableOfflineException(instance, tableId);
+ }
+ binnedRanges.clear();
+ log.warn("Unable to locate bins for specified ranges. Retrying.");
+ UtilWaitThread.sleep(100 + (int) (Math.random() * 100)); // sleep randomly between 100 and 200 ms
+ tl.invalidateCache();
+ }
+ }
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ ArrayList<org.apache.accumulo.core.client.mapred.RangeInputSplit> splits = new ArrayList<org.apache.accumulo.core.client.mapred.RangeInputSplit>(
+ ranges.size());
+ HashMap<Range,ArrayList<String>> splitsToAdd = null;
+
+ if (!autoAdjust)
+ splitsToAdd = new HashMap<Range,ArrayList<String>>();
+
+ HashMap<String,String> hostNameCache = new HashMap<String,String>();
+
+ for (Entry<String,Map<KeyExtent,List<Range>>> tserverBin : binnedRanges.entrySet()) {
+ String ip = tserverBin.getKey().split(":", 2)[0];
+ String location = hostNameCache.get(ip);
+ if (location == null) {
+ InetAddress inetAddress = InetAddress.getByName(ip);
+ location = inetAddress.getHostName();
+ hostNameCache.put(ip, location);
+ }
+
+ for (Entry<KeyExtent,List<Range>> extentRanges : tserverBin.getValue().entrySet()) {
+ Range ke = extentRanges.getKey().toDataRange();
+ for (Range r : extentRanges.getValue()) {
+ if (autoAdjust) {
+ // divide ranges into smaller ranges, based on the tablets
+ splits.add(new org.apache.accumulo.core.client.mapred.RangeInputSplit(ke.clip(r), new String[] {location}));
+ } else {
+ // don't divide ranges
+ ArrayList<String> locations = splitsToAdd.get(r);
+ if (locations == null)
+ locations = new ArrayList<String>(1);
+ locations.add(location);
+ splitsToAdd.put(r, locations);
+ }
+ }
+ }
+ }
+
+ if (!autoAdjust)
+ for (Entry<Range,ArrayList<String>> entry : splitsToAdd.entrySet())
+ splits.add(new org.apache.accumulo.core.client.mapred.RangeInputSplit(entry.getKey(), entry.getValue().toArray(new String[0])));
+
+ for (org.apache.accumulo.core.client.mapred.RangeInputSplit split : splits) {
+ split.setTable(tableName);
+ split.setOffline(offline);
+ split.setIsolatedScan(isolated);
+ split.setUsesLocalIterators(localIterators);
+ split.setMockInstance(mockInstance);
+ split.setFetchedColumns(fetchedColumns);
+ split.setPrincipal(principal);
+ split.setToken(token);
+ split.setInstanceName(instance.getInstanceName());
+ split.setZooKeepers(instance.getZooKeepers());
+ split.setAuths(auths);
+ split.setIterators(iterators);
+ split.setLogLevel(logLevel);
+ }
+
+ return splits.toArray(new InputSplit[splits.size()]);
+ }
+
+ /**
+ * @deprecated since 1.5.2; Use {@link org.apache.accumulo.core.client.mapred.RangeInputSplit} instead.
+ * @see org.apache.accumulo.core.client.mapred.RangeInputSplit
+ */
+ @Deprecated
+ public static class RangeInputSplit extends org.apache.accumulo.core.client.mapred.RangeInputSplit {
+ public RangeInputSplit() {
+ super();
+ }
+
+ public RangeInputSplit(Range range, String[] locations) {
+ super(range, locations);
+ }
+ }
+}
[23/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --cc minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
index a366c16,0000000..e2b2f83
mode 100644,000000..100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
@@@ -1,392 -1,0 +1,382 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.minicluster;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.gc.SimpleGarbageCollector;
+import org.apache.accumulo.server.master.Master;
+import org.apache.accumulo.server.tabletserver.TabletServer;
+import org.apache.accumulo.server.util.Initialize;
+import org.apache.accumulo.server.util.PortUtils;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.start.Main;
+import org.apache.zookeeper.server.ZooKeeperServerMain;
+
+/**
+ * A utility class that will create Zookeeper and Accumulo processes that write all of their data to a single local directory. This class makes it easy to test
+ * code against a real Accumulo instance. Its much more accurate for testing than MockAccumulo, but much slower than MockAccumulo.
+ *
+ * @since 1.5.0
+ */
+public class MiniAccumuloCluster {
+
+ private static final String INSTANCE_SECRET = "DONTTELL";
+ private static final String INSTANCE_NAME = "miniInstance";
+
+ private static class LogWriter extends Thread {
+ private BufferedReader in;
+ private BufferedWriter out;
+
- /**
- * @throws IOException
- */
+ public LogWriter(InputStream stream, File logFile) throws IOException {
+ this.setDaemon(true);
+ this.in = new BufferedReader(new InputStreamReader(stream, Constants.UTF8));
+ out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(logFile), Constants.UTF8));
+
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ flush();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }, 1000, 1000);
+ }
+
+ public synchronized void flush() throws IOException {
+ if (out != null)
+ out.flush();
+ }
+
+ @Override
+ public void run() {
+ String line;
+
+ try {
+ while ((line = in.readLine()) != null) {
+ out.append(line);
+ out.append("\n");
+ }
+
+ synchronized (this) {
+ out.close();
+ out = null;
+ in.close();
+ }
+
+ } catch (IOException e) {}
+ }
+ }
+
+ private File libDir;
+ private File libExtDir;
+ private File confDir;
+ private File zooKeeperDir;
+ private File accumuloDir;
+ private File zooCfgFile;
+ private File logDir;
+ private File walogDir;
+
+ private Process zooKeeperProcess;
+ private Process masterProcess;
+ private Process gcProcess;
+
+ private int zooKeeperPort;
+
+ private List<LogWriter> logWriters = new ArrayList<MiniAccumuloCluster.LogWriter>();
+
+ private MiniAccumuloConfig config;
+ private Process[] tabletServerProcesses;
+
+ private Process exec(Class<? extends Object> clazz, String... args) throws IOException {
+ String javaHome = System.getProperty("java.home");
+ String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
+ String classpath = System.getProperty("java.class.path");
+
+ classpath = confDir.getAbsolutePath() + File.pathSeparator + classpath;
+
+ String className = clazz.getCanonicalName();
+
+ ArrayList<String> argList = new ArrayList<String>();
+
+ argList.addAll(Arrays.asList(javaBin, "-cp", classpath, "-Xmx128m", "-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75",
+ "-Dapple.awt.UIElement=true", Main.class.getName(), className));
+
+ argList.addAll(Arrays.asList(args));
+
+ ProcessBuilder builder = new ProcessBuilder(argList);
+
+ builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
+ builder.environment().put("ACCUMULO_LOG_DIR", logDir.getAbsolutePath());
+
+ // if we're running under accumulo.start, we forward these env vars
+ String env = System.getenv("HADOOP_PREFIX");
+ if (env != null)
+ builder.environment().put("HADOOP_PREFIX", env);
+ env = System.getenv("ZOOKEEPER_HOME");
+ if (env != null)
+ builder.environment().put("ZOOKEEPER_HOME", env);
+
+ Process process = builder.start();
+
+ LogWriter lw;
+ lw = new LogWriter(process.getErrorStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".err"));
+ logWriters.add(lw);
+ lw.start();
+ lw = new LogWriter(process.getInputStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".out"));
+ logWriters.add(lw);
+ lw.start();
+
+ return process;
+ }
+
+ private void appendProp(Writer fileWriter, Property key, String value, Map<String,String> siteConfig) throws IOException {
+ appendProp(fileWriter, key.getKey(), value, siteConfig);
+ }
+
+ private void appendProp(Writer fileWriter, String key, String value, Map<String,String> siteConfig) throws IOException {
+ if (!siteConfig.containsKey(key))
+ fileWriter.append("<property><name>" + key + "</name><value>" + value + "</value></property>\n");
+ }
+
+ /**
+ * Sets a given key with a random port for the value on the site config if it doesn't already exist.
+ */
+ private void mergePropWithRandomPort(Map<String,String> siteConfig, String key) {
+ if (!siteConfig.containsKey(key)) {
+ siteConfig.put(key, "0");
+ }
+ }
+
+ /**
+ *
+ * @param dir
+ * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
+ * and Junit provide methods for creating temporary directories.
+ * @param rootPassword
+ * Initial root password for instance.
- * @throws IOException
+ */
+ public MiniAccumuloCluster(File dir, String rootPassword) throws IOException {
+ this(new MiniAccumuloConfig(dir, rootPassword));
+ }
+
+ /**
+ * @param config
+ * initial configuration
- * @throws IOException
+ */
+
+ public MiniAccumuloCluster(MiniAccumuloConfig config) throws IOException {
+
+ if (config.getDir().exists() && !config.getDir().isDirectory())
+ throw new IllegalArgumentException("Must pass in directory, " + config.getDir() + " is a file");
+
+ if (config.getDir().exists() && config.getDir().list().length != 0)
+ throw new IllegalArgumentException("Directory " + config.getDir() + " is not empty");
+
+ this.config = config;
+
+ libDir = new File(config.getDir(), "lib");
+ libExtDir = new File(libDir, "ext");
+ confDir = new File(config.getDir(), "conf");
+ accumuloDir = new File(config.getDir(), "accumulo");
+ zooKeeperDir = new File(config.getDir(), "zookeeper");
+ logDir = new File(config.getDir(), "logs");
+ walogDir = new File(config.getDir(), "walogs");
+
+ confDir.mkdirs();
+ accumuloDir.mkdirs();
+ zooKeeperDir.mkdirs();
+ logDir.mkdirs();
+ walogDir.mkdirs();
+ libDir.mkdirs();
+
+ // Avoid the classloader yelling that the general.dynamic.classpaths value is invalid because
+ // $ACCUMULO_HOME/lib/ext isn't defined.
+ libExtDir.mkdirs();
+
+ zooKeeperPort = PortUtils.getRandomFreePort();
+
+ File siteFile = new File(confDir, "accumulo-site.xml");
+
+ OutputStreamWriter fileWriter = new OutputStreamWriter(new FileOutputStream(siteFile), Constants.UTF8);
+ fileWriter.append("<configuration>\n");
+
+ HashMap<String,String> siteConfig = new HashMap<String,String>(config.getSiteConfig());
+
+ appendProp(fileWriter, Property.INSTANCE_DFS_URI, "file:///", siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_DFS_DIR, accumuloDir.getAbsolutePath(), siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_ZK_HOST, "localhost:" + zooKeeperPort, siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_SECRET, INSTANCE_SECRET, siteConfig);
+ appendProp(fileWriter, Property.TSERV_PORTSEARCH, "true", siteConfig);
+ appendProp(fileWriter, Property.LOGGER_DIR, walogDir.getAbsolutePath(), siteConfig);
+ appendProp(fileWriter, Property.TSERV_DATACACHE_SIZE, "10M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_INDEXCACHE_SIZE, "10M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_MAXMEM, "50M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_WALOG_MAX_SIZE, "100M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_NATIVEMAP_ENABLED, "false", siteConfig);
+ appendProp(fileWriter, Property.TRACE_TOKEN_PROPERTY_PREFIX + ".password", config.getRootPassword(), siteConfig);
+ appendProp(fileWriter, Property.GC_CYCLE_DELAY, "4s", siteConfig);
+ appendProp(fileWriter, Property.GC_CYCLE_START, "0s", siteConfig);
+ mergePropWithRandomPort(siteConfig, Property.MASTER_CLIENTPORT.getKey());
+ mergePropWithRandomPort(siteConfig, Property.TRACE_PORT.getKey());
+ mergePropWithRandomPort(siteConfig, Property.TSERV_CLIENTPORT.getKey());
+ mergePropWithRandomPort(siteConfig, Property.MONITOR_PORT.getKey());
+ mergePropWithRandomPort(siteConfig, Property.GC_PORT.getKey());
+
+ // since there is a small amount of memory, check more frequently for majc... setting may not be needed in 1.5
+ appendProp(fileWriter, Property.TSERV_MAJC_DELAY, "3", siteConfig);
+
+ // ACCUMULO-1472 -- Use the classpath, not what might be installed on the system.
+ // We have to set *something* here, otherwise the AccumuloClassLoader will default to pulling from
+ // environment variables (e.g. ACCUMULO_HOME, HADOOP_HOME/PREFIX) which will result in multiple copies
+ // of artifacts on the classpath as they'll be provided by the invoking application
+ appendProp(fileWriter, Property.GENERAL_CLASSPATHS, libDir.getAbsolutePath() + "/[^.].*.jar", siteConfig);
+ appendProp(fileWriter, Property.GENERAL_DYNAMIC_CLASSPATHS, libExtDir.getAbsolutePath() + "/[^.].*.jar", siteConfig);
+
+ for (Entry<String,String> entry : siteConfig.entrySet())
+ fileWriter.append("<property><name>" + entry.getKey() + "</name><value>" + entry.getValue() + "</value></property>\n");
+ fileWriter.append("</configuration>\n");
+ fileWriter.close();
+
+ zooCfgFile = new File(confDir, "zoo.cfg");
+ fileWriter = new OutputStreamWriter(new FileOutputStream(zooCfgFile), Constants.UTF8);
+
+ // zookeeper uses Properties to read its config, so use that to write in order to properly escape things like Windows paths
+ Properties zooCfg = new Properties();
+ zooCfg.setProperty("tickTime", "1000");
+ zooCfg.setProperty("initLimit", "10");
+ zooCfg.setProperty("syncLimit", "5");
+ zooCfg.setProperty("clientPort", zooKeeperPort + "");
+ zooCfg.setProperty("maxClientCnxns", "100");
+ zooCfg.setProperty("dataDir", zooKeeperDir.getAbsolutePath());
+ zooCfg.store(fileWriter, null);
+
+ fileWriter.close();
+
+ }
+
+ /**
+ * Starts Accumulo and Zookeeper processes. Can only be called once.
+ *
- * @throws IOException
- * @throws InterruptedException
+ * @throws IllegalStateException
+ * if already started
+ */
+
+ public void start() throws IOException, InterruptedException {
+ if (zooKeeperProcess != null)
+ throw new IllegalStateException("Already started");
+
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ try {
+ MiniAccumuloCluster.this.stop();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+
+ zooKeeperProcess = exec(Main.class, ZooKeeperServerMain.class.getName(), zooCfgFile.getAbsolutePath());
+
+ // sleep a little bit to let zookeeper come up before calling init, seems to work better
+ UtilWaitThread.sleep(250);
+
+ Process initProcess = exec(Initialize.class, "--instance-name", INSTANCE_NAME, "--password", config.getRootPassword());
+ int ret = initProcess.waitFor();
+ if (ret != 0) {
+ throw new RuntimeException("Initialize process returned " + ret + ". Check the logs in " + logDir + " for errors.");
+ }
+
+ tabletServerProcesses = new Process[config.getNumTservers()];
+ for (int i = 0; i < config.getNumTservers(); i++) {
+ tabletServerProcesses[i] = exec(TabletServer.class);
+ }
+
+ masterProcess = exec(Master.class);
+
+ gcProcess = exec(SimpleGarbageCollector.class);
+ }
+
+ /**
+ * @return Accumulo instance name
+ */
+
+ public String getInstanceName() {
+ return INSTANCE_NAME;
+ }
+
+ /**
+ * @return zookeeper connection string
+ */
+
+ public String getZooKeepers() {
+ return "localhost:" + zooKeeperPort;
+ }
+
+ /**
+ * Stops Accumulo and Zookeeper processes. If stop is not called, there is a shutdown hook that is setup to kill the processes. Howerver its probably best to
+ * call stop in a finally block as soon as possible.
- *
- * @throws IOException
- * @throws InterruptedException
+ */
+
+ public void stop() throws IOException, InterruptedException {
+ if (zooKeeperProcess != null) {
+ zooKeeperProcess.destroy();
+ zooKeeperProcess.waitFor();
+ }
+ if (masterProcess != null) {
+ masterProcess.destroy();
+ masterProcess.waitFor();
+ }
+ if (tabletServerProcesses != null) {
+ for (Process tserver : tabletServerProcesses) {
+ tserver.destroy();
+ tserver.waitFor();
+ }
+ }
+
+ for (LogWriter lw : logWriters)
+ lw.flush();
+
+ if (gcProcess != null) {
+ gcProcess.destroy();
+ gcProcess.waitFor();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/proxy/src/test/java/org/apache/accumulo/proxy/TestProxyReadWrite.java
----------------------------------------------------------------------
diff --cc proxy/src/test/java/org/apache/accumulo/proxy/TestProxyReadWrite.java
index 99a3218,0000000..c0049a0
mode 100644,000000..100644
--- a/proxy/src/test/java/org/apache/accumulo/proxy/TestProxyReadWrite.java
+++ b/proxy/src/test/java/org/apache/accumulo/proxy/TestProxyReadWrite.java
@@@ -1,488 -1,0 +1,478 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.proxy;
+
+import static org.junit.Assert.assertEquals;
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.proxy.thrift.BatchScanOptions;
+import org.apache.accumulo.proxy.thrift.ColumnUpdate;
+import org.apache.accumulo.proxy.thrift.IteratorSetting;
+import org.apache.accumulo.proxy.thrift.Key;
+import org.apache.accumulo.proxy.thrift.KeyValue;
+import org.apache.accumulo.proxy.thrift.Range;
+import org.apache.accumulo.proxy.thrift.ScanColumn;
+import org.apache.accumulo.proxy.thrift.ScanOptions;
+import org.apache.accumulo.proxy.thrift.ScanResult;
+import org.apache.accumulo.proxy.thrift.TimeType;
+import org.apache.thrift.protocol.TCompactProtocol;
+import org.apache.thrift.server.TServer;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestProxyReadWrite {
+ protected static TServer proxy;
+ protected static Thread thread;
+ protected static TestProxyClient tpc;
+ protected static ByteBuffer userpass;
+ protected static final int port = 10194;
+ protected static final String testtable = "testtable";
+
+ @SuppressWarnings("serial")
+ @BeforeClass
+ public static void setup() throws Exception {
+ Properties prop = new Properties();
+ prop.setProperty("useMockInstance", "true");
+ prop.put("tokenClass", PasswordToken.class.getName());
+
+ proxy = Proxy.createProxyServer(Class.forName("org.apache.accumulo.proxy.thrift.AccumuloProxy"), Class.forName("org.apache.accumulo.proxy.ProxyServer"),
+ port, TCompactProtocol.Factory.class, prop);
+ thread = new Thread() {
+ @Override
+ public void run() {
+ proxy.serve();
+ }
+ };
+ thread.start();
+ tpc = new TestProxyClient("localhost", port);
+ userpass = tpc.proxy().login("root", new TreeMap<String, String>() {{put("password",""); }});
+ }
+
+ @AfterClass
+ public static void tearDown() throws InterruptedException {
+ proxy.stop();
+ thread.join();
+ }
+
+ @Before
+ public void makeTestTable() throws Exception {
+ tpc.proxy().createTable(userpass, testtable, true, TimeType.MILLIS);
+ }
+
+ @After
+ public void deleteTestTable() throws Exception {
+ tpc.proxy().deleteTable(userpass, testtable);
+ }
+
+ private static void addMutation(Map<ByteBuffer,List<ColumnUpdate>> mutations, String row, String cf, String cq, String value) {
+ ColumnUpdate update = new ColumnUpdate(ByteBuffer.wrap(cf.getBytes()), ByteBuffer.wrap(cq.getBytes()));
+ update.setValue(value.getBytes());
+ mutations.put(ByteBuffer.wrap(row.getBytes()), Collections.singletonList(update));
+ }
+
+ private static void addMutation(Map<ByteBuffer,List<ColumnUpdate>> mutations, String row, String cf, String cq, String vis, String value) {
+ ColumnUpdate update = new ColumnUpdate(ByteBuffer.wrap(cf.getBytes()), ByteBuffer.wrap(cq.getBytes()));
+ update.setValue(value.getBytes());
+ update.setColVisibility(vis.getBytes());
+ mutations.put(ByteBuffer.wrap(row.getBytes()), Collections.singletonList(update));
+ }
+
+ /**
+ * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Set a range so only the entries between -Inf...5 come back (there should be
+ * 50,000)
- *
- * @throws Exception
+ */
+ @Test
+ public void readWriteBatchOneShotWithRange() throws Exception {
+ int maxInserts = 100000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ for (int i = 0; i < maxInserts; i++) {
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+ tpc.proxy().updateAndFlush(userpass, testtable, mutations);
+ mutations.clear();
+ }
+ }
+
+ Key stop = new Key();
+ stop.setRow("5".getBytes());
+ BatchScanOptions options = new BatchScanOptions();
+ options.ranges = Collections.singletonList(new Range(null, false, stop, false));
+ String cookie = tpc.proxy().createBatchScanner(userpass, testtable, options);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ i += kvList.getResultsSize();
+ hasNext = kvList.isMore();
+ }
+ assertEquals(i, 50000);
+ }
+
+ /**
+ * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Set a columnFamily so only the entries with specified column family come back (there should be
+ * 50,000)
- *
- * @throws Exception
+ */
+ @Test
+ public void readWriteBatchOneShotWithColumnFamilyOnly() throws Exception {
+ int maxInserts = 100000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ for (int i = 0; i < maxInserts; i++) {
+
+ addMutation(mutations, String.format(format, i), "cf" + (i % 2) , "cq" + (i % 2), Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+ tpc.proxy().updateAndFlush(userpass, testtable, mutations);
+ mutations.clear();
+ }
+ }
+
+ BatchScanOptions options = new BatchScanOptions();
+
+ ScanColumn sc = new ScanColumn();
+ sc.colFamily = ByteBuffer.wrap("cf0".getBytes());
+
+ options.columns = Collections.singletonList(sc);
+ String cookie = tpc.proxy().createBatchScanner(userpass, testtable, options);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ i += kvList.getResultsSize();
+ hasNext = kvList.isMore();
+ }
+ assertEquals(i, 50000);
+ }
+
+
+ /**
+ * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Set a columnFamily + columnQualififer so only the entries with specified column
+ * come back (there should be 50,000)
- *
- * @throws Exception
+ */
+ @Test
+ public void readWriteBatchOneShotWithFullColumn() throws Exception {
+ int maxInserts = 100000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ for (int i = 0; i < maxInserts; i++) {
+
+ addMutation(mutations, String.format(format, i), "cf" + (i % 2) , "cq" + (i % 2), Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+ tpc.proxy().updateAndFlush(userpass, testtable, mutations);
+ mutations.clear();
+ }
+ }
+
+ BatchScanOptions options = new BatchScanOptions();
+
+ ScanColumn sc = new ScanColumn();
+ sc.colFamily = ByteBuffer.wrap("cf0".getBytes());
+ sc.colQualifier = ByteBuffer.wrap("cq0".getBytes());
+
+ options.columns = Collections.singletonList(sc);
+ String cookie = tpc.proxy().createBatchScanner(userpass, testtable, options);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ i += kvList.getResultsSize();
+ hasNext = kvList.isMore();
+ }
+ assertEquals(i, 50000);
+ }
+
+
+ /**
+ * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Filter the results so only the even numbers come back.
- *
- * @throws Exception
+ */
+ @Test
+ public void readWriteBatchOneShotWithFilterIterator() throws Exception {
+ int maxInserts = 10000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ for (int i = 0; i < maxInserts; i++) {
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+ tpc.proxy().updateAndFlush(userpass, testtable, mutations);
+ mutations.clear();
+ }
+
+ }
+
+ String regex = ".*[02468]";
+
+ org.apache.accumulo.core.client.IteratorSetting is = new org.apache.accumulo.core.client.IteratorSetting(50, regex, RegExFilter.class);
+ RegExFilter.setRegexs(is, regex, null, null, null, false);
+
+ IteratorSetting pis = Util.iteratorSetting2ProxyIteratorSetting(is);
+ ScanOptions opts = new ScanOptions();
+ opts.iterators = Collections.singletonList(pis);
+ String cookie = tpc.proxy().createScanner(userpass, testtable, opts);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ for (KeyValue kv : kvList.getResults()) {
+ assertEquals(Integer.parseInt(new String(kv.getKey().getRow())), i);
+
+ i += 2;
+ }
+ hasNext = kvList.isMore();
+ }
+ }
+
+ @Test
+ public void readWriteOneShotWithRange() throws Exception {
+ int maxInserts = 100000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ for (int i = 0; i < maxInserts; i++) {
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+ tpc.proxy().updateAndFlush(userpass, testtable, mutations);
+ mutations.clear();
+ }
+ }
+
+ Key stop = new Key();
+ stop.setRow("5".getBytes());
+ ScanOptions opts = new ScanOptions();
+ opts.range = new Range(null, false, stop, false);
+ String cookie = tpc.proxy().createScanner(userpass, testtable, opts);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ i += kvList.getResultsSize();
+ hasNext = kvList.isMore();
+ }
+ assertEquals(i, 50000);
+ }
+
+ /**
+ * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Filter the results so only the even numbers come back.
- *
- * @throws Exception
+ */
+ @Test
+ public void readWriteOneShotWithFilterIterator() throws Exception {
+ int maxInserts = 10000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ for (int i = 0; i < maxInserts; i++) {
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+
+ tpc.proxy().updateAndFlush(userpass, testtable, mutations);
+ mutations.clear();
+
+ }
+
+ }
+
+ String regex = ".*[02468]";
+
+ org.apache.accumulo.core.client.IteratorSetting is = new org.apache.accumulo.core.client.IteratorSetting(50, regex, RegExFilter.class);
+ RegExFilter.setRegexs(is, regex, null, null, null, false);
+
+ IteratorSetting pis = Util.iteratorSetting2ProxyIteratorSetting(is);
+ ScanOptions opts = new ScanOptions();
+ opts.iterators = Collections.singletonList(pis);
+ String cookie = tpc.proxy().createScanner(userpass, testtable, opts);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ for (KeyValue kv : kvList.getResults()) {
+ assertEquals(Integer.parseInt(new String(kv.getKey().getRow())), i);
+
+ i += 2;
+ }
+ hasNext = kvList.isMore();
+ }
+ }
+
+ // @Test
+ // This test takes kind of a long time. Enable it if you think you may have memory issues.
+ public void manyWritesAndReads() throws Exception {
+ int maxInserts = 1000000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$06d";
+ String writer = tpc.proxy().createWriter(userpass, testtable, null);
+ for (int i = 0; i < maxInserts; i++) {
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+
+ tpc.proxy().update(writer, mutations);
+ mutations.clear();
+
+ }
+
+ }
+
+ tpc.proxy().flush(writer);
+ tpc.proxy().closeWriter(writer);
+
+ String cookie = tpc.proxy().createScanner(userpass, testtable, null);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ for (KeyValue kv : kvList.getResults()) {
+ assertEquals(Integer.parseInt(new String(kv.getKey().getRow())), i);
+ i++;
+ }
+ hasNext = kvList.isMore();
+ if (hasNext)
+ assertEquals(k, kvList.getResults().size());
+ }
+ assertEquals(maxInserts, i);
+ }
+
+ @Test
+ public void asynchReadWrite() throws Exception {
+ int maxInserts = 10000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ String writer = tpc.proxy().createWriter(userpass, testtable, null);
+ for (int i = 0; i < maxInserts; i++) {
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+ tpc.proxy().update(writer, mutations);
+ mutations.clear();
+ }
+ }
+
+ tpc.proxy().flush(writer);
+ tpc.proxy().closeWriter(writer);
+
+ String regex = ".*[02468]";
+
+ org.apache.accumulo.core.client.IteratorSetting is = new org.apache.accumulo.core.client.IteratorSetting(50, regex, RegExFilter.class);
+ RegExFilter.setRegexs(is, regex, null, null, null, false);
+
+ IteratorSetting pis = Util.iteratorSetting2ProxyIteratorSetting(is);
+ ScanOptions opts = new ScanOptions();
+ opts.iterators = Collections.singletonList(pis);
+ String cookie = tpc.proxy().createScanner(userpass, testtable, opts);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ int numRead = 0;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ for (KeyValue kv : kvList.getResults()) {
+ assertEquals(i, Integer.parseInt(new String(kv.getKey().getRow())));
+ numRead++;
+ i += 2;
+ }
+ hasNext = kvList.isMore();
+ }
+ assertEquals(maxInserts / 2, numRead);
+ }
+
+ @Test
+ public void testVisibility() throws Exception {
+
+ Set<ByteBuffer> auths = new HashSet<ByteBuffer>();
+ auths.add(ByteBuffer.wrap("even".getBytes()));
+ tpc.proxy().changeUserAuthorizations(userpass, "root", auths);
+
+ int maxInserts = 10000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ String writer = tpc.proxy().createWriter(userpass, testtable, null);
+ for (int i = 0; i < maxInserts; i++) {
+ if (i % 2 == 0)
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, "even", Util.randString(10));
+ else
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, "odd", Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+ tpc.proxy().update(writer, mutations);
+ mutations.clear();
+ }
+ }
+
+ tpc.proxy().flush(writer);
+ tpc.proxy().closeWriter(writer);
+ ScanOptions opts = new ScanOptions();
+ opts.authorizations = auths;
+ String cookie = tpc.proxy().createScanner(userpass, testtable, opts);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ int numRead = 0;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ for (KeyValue kv : kvList.getResults()) {
+ assertEquals(Integer.parseInt(new String(kv.getKey().getRow())), i);
+ i += 2;
+ numRead++;
+ }
+ hasNext = kvList.isMore();
+
+ }
+ assertEquals(maxInserts / 2, numRead);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java
index 442294f,0000000..05806ca
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java
+++ b/server/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java
@@@ -1,30 -1,0 +1,27 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.conf;
+
+import org.apache.accumulo.server.client.HdfsZooInstance;
+
+public class ConfigSanityCheck {
+
- /**
- * @param args
- */
+ public static void main(String[] args) {
+ new ServerConfiguration(HdfsZooInstance.getInstance()).getConfiguration();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
index 4f9d33a,0000000..01626ad
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
+++ b/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
@@@ -1,191 -1,0 +1,190 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.logger;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.tabletserver.log.DfsLogger;
+import org.apache.accumulo.server.tabletserver.log.MultiReader;
+import org.apache.accumulo.server.trace.TraceFileSystem;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+
+public class LogReader {
+
+ static class Opts extends Help {
+ @Parameter(names = "-r", description = "print only mutations associated with the given row")
+ String row;
+ @Parameter(names = "-m", description = "limit the number of mutations printed per row")
+ int maxMutations = 5;
+ @Parameter(names = "-t", description = "print only mutations that fall within the given key extent")
+ String extent;
+ @Parameter(names = "-p", description = "search for a row that matches the given regex")
+ String regexp;
+ @Parameter(description = "<logfile> { <logfile> ... }")
+ List<String> files = new ArrayList<String>();
+ }
+
+ /**
+ * Dump a Log File (Map or Sequence) to stdout. Will read from HDFS or local file system.
+ *
+ * @param args
+ * - first argument is the file to print
- * @throws IOException
+ */
+ public static void main(String[] args) throws IOException {
+ Opts opts = new Opts();
+ opts.parseArgs(LogReader.class.getName(), args);
+ Configuration conf = CachedConfiguration.getInstance();
+ FileSystem fs = TraceFileSystem.wrap(FileUtil.getFileSystem(conf, ServerConfiguration.getSiteConfiguration()));
+ FileSystem local = TraceFileSystem.wrap(FileSystem.getLocal(conf));
+
+ Matcher rowMatcher = null;
+ KeyExtent ke = null;
+ Text row = null;
+ if (opts.files.isEmpty()) {
+ new JCommander(opts).usage();
+ return;
+ }
+ if (opts.row != null)
+ row = new Text(opts.row);
+ if (opts.extent != null) {
+ String sa[] = opts.extent.split(";");
+ ke = new KeyExtent(new Text(sa[0]), new Text(sa[1]), new Text(sa[2]));
+ }
+ if (opts.regexp != null) {
+ Pattern pattern = Pattern.compile(opts.regexp);
+ rowMatcher = pattern.matcher("");
+ }
+
+ Set<Integer> tabletIds = new HashSet<Integer>();
+
+ for (String file : opts.files) {
+
+ Map<String, String> meta = new HashMap<String, String>();
+ Path path = new Path(file);
+ LogFileKey key = new LogFileKey();
+ LogFileValue value = new LogFileValue();
+
+ if (fs.isFile(path)) {
+ // read log entries from a simple hdfs file
+ FSDataInputStream f = DfsLogger.readHeader(fs, path, meta);
+ try {
+ while (true) {
+ try {
+ key.readFields(f);
+ value.readFields(f);
+ } catch (EOFException ex) {
+ break;
+ }
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
+ }
+ } finally {
+ f.close();
+ }
+ } else if (local.isFile(path)) {
+ // read log entries from a simple file
+ FSDataInputStream f = DfsLogger.readHeader(fs, path, meta);
+ try {
+ while (true) {
+ try {
+ key.readFields(f);
+ value.readFields(f);
+ } catch (EOFException ex) {
+ break;
+ }
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
+ }
+ } finally {
+ f.close();
+ }
+ } else {
+ // read the log entries sorted in a map file
+ MultiReader input = new MultiReader(fs, conf, file);
+ while (input.next(key, value)) {
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
+ }
+ }
+ }
+ }
+
+ public static void printLogEvent(LogFileKey key, LogFileValue value, Text row, Matcher rowMatcher, KeyExtent ke, Set<Integer> tabletIds, int maxMutations) {
+
+ if (ke != null) {
+ if (key.event == LogEvents.DEFINE_TABLET) {
+ if (key.tablet.equals(ke)) {
+ tabletIds.add(key.tid);
+ } else {
+ return;
+ }
+ } else if (!tabletIds.contains(key.tid)) {
+ return;
+ }
+ }
+
+ if (row != null || rowMatcher != null) {
+ if (key.event == LogEvents.MUTATION || key.event == LogEvents.MANY_MUTATIONS) {
+ boolean found = false;
+ for (Mutation m : value.mutations) {
+ if (row != null && new Text(m.getRow()).equals(row)) {
+ found = true;
+ break;
+ }
+
+ if (rowMatcher != null) {
+ rowMatcher.reset(new String(m.getRow(), Constants.UTF8));
+ if (rowMatcher.matches()) {
+ found = true;
+ break;
+ }
+ }
+ }
+
+ if (!found)
+ return;
+ } else {
+ return;
+ }
+
+ }
+
+ System.out.println(key);
+ System.out.println(LogFileValue.format(value, maxMutations));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java
index 4930bc2,0000000..e14008a
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java
@@@ -1,152 -1,0 +1,144 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.balancer;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+import java.util.SortedMap;
+
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.master.thrift.TableInfo;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletMigration;
+import org.apache.thrift.TException;
+
+/**
+ * A chaotic load balancer used for testing. It constantly shuffles tablets, preventing them from resting in a single location for very long. This is not
+ * designed for performance, do not use on production systems. I'm calling it the LokiLoadBalancer.
+ */
+public class ChaoticLoadBalancer extends TabletBalancer {
+ Random r = new Random();
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.master.balancer.TabletBalancer#getAssignments(java.util.SortedMap, java.util.Map, java.util.Map)
- */
+ @Override
+ public void getAssignments(SortedMap<TServerInstance,TabletServerStatus> current, Map<KeyExtent,TServerInstance> unassigned,
+ Map<KeyExtent,TServerInstance> assignments) {
+ long total = assignments.size() + unassigned.size();
+ long avg = (long) Math.ceil(((double) total) / current.size());
+ Map<TServerInstance,Long> toAssign = new HashMap<TServerInstance,Long>();
+ List<TServerInstance> tServerArray = new ArrayList<TServerInstance>();
+ for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
+ long numTablets = 0;
+ for (TableInfo ti : e.getValue().getTableMap().values()) {
+ numTablets += ti.tablets;
+ }
+ if (numTablets < avg) {
+ tServerArray.add(e.getKey());
+ toAssign.put(e.getKey(), avg - numTablets);
+ }
+ }
+
+ for (KeyExtent ke : unassigned.keySet())
+ {
+ int index = r.nextInt(tServerArray.size());
+ TServerInstance dest = tServerArray.get(index);
+ assignments.put(ke, dest);
+ long remaining = toAssign.get(dest).longValue() - 1;
+ if (remaining == 0) {
+ tServerArray.remove(index);
+ toAssign.remove(dest);
+ } else {
+ toAssign.put(dest, remaining);
+ }
+ }
+ }
+
+ /**
+ * Will balance randomly, maintaining distribution
+ */
+ @Override
+ public long balance(SortedMap<TServerInstance,TabletServerStatus> current, Set<KeyExtent> migrations, List<TabletMigration> migrationsOut) {
+ Map<TServerInstance,Long> numTablets = new HashMap<TServerInstance,Long>();
+ List<TServerInstance> underCapacityTServer = new ArrayList<TServerInstance>();
+
+ if (!migrations.isEmpty())
+ return 100;
+
+ boolean moveMetadata = r.nextInt(4) == 0;
+ long totalTablets = 0;
+ for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
+ long tabletCount = 0;
+ for (TableInfo ti : e.getValue().getTableMap().values()) {
+ tabletCount += ti.tablets;
+ }
+ numTablets.put(e.getKey(), tabletCount);
+ underCapacityTServer.add(e.getKey());
+ totalTablets += tabletCount;
+ }
+ // totalTablets is fuzzy due to asynchronicity of the stats
+ // *1.2 to handle fuzziness, and prevent locking for 'perfect' balancing scenarios
+ long avg = (long) Math.ceil(((double) totalTablets) / current.size() * 1.2);
+
+ for (Entry<TServerInstance, TabletServerStatus> e : current.entrySet())
+ {
+ for (String table : e.getValue().getTableMap().keySet())
+ {
+ if (!moveMetadata && "!METADATA".equals(table))
+ continue;
+ try {
+ for (TabletStats ts : getOnlineTabletsForTable(e.getKey(), table)) {
+ KeyExtent ke = new KeyExtent(ts.extent);
+ int index = r.nextInt(underCapacityTServer.size());
+ TServerInstance dest = underCapacityTServer.get(index);
+ if (dest.equals(e.getKey()))
+ continue;
+ migrationsOut.add(new TabletMigration(ke, e.getKey(), dest));
+ if (numTablets.put(dest, numTablets.get(dest) + 1) > avg)
+ underCapacityTServer.remove(index);
+ if (numTablets.put(e.getKey(), numTablets.get(e.getKey()) - 1) <= avg && !underCapacityTServer.contains(e.getKey()))
+ underCapacityTServer.add(e.getKey());
+
+ // We can get some craziness with only 1 tserver, so lets make sure there's always an option!
+ if (underCapacityTServer.isEmpty())
+ underCapacityTServer.addAll(numTablets.keySet());
+ }
+ } catch (ThriftSecurityException e1) {
+ // Shouldn't happen, but carry on if it does
+ e1.printStackTrace();
+ } catch (TException e1) {
+ // Shouldn't happen, but carry on if it does
+ e1.printStackTrace();
+ }
+ }
+ }
+
+ return 100;
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.server.master.balancer.TabletBalancer#init(org.apache.accumulo.server.conf.ServerConfiguration)
- */
+ @Override
+ public void init(ServerConfiguration conf) {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
index d6dce2f,0000000..69387d3
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
@@@ -1,150 -1,0 +1,148 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.balancer;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletMigration;
+import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransportException;
+
+public abstract class TabletBalancer {
+
+ private static final Logger log = Logger.getLogger(TabletBalancer.class);
+
+ protected ServerConfiguration configuration;
+
+ /**
+ * Initialize the TabletBalancer. This gives the balancer the opportunity to read the configuration.
+ */
+ public void init(ServerConfiguration conf) {
+ configuration = conf;
+ }
+
+ /**
+ * Assign tablets to tablet servers. This method is called whenever the master finds tablets that are unassigned.
+ *
+ * @param current
+ * The current table-summary state of all the online tablet servers. Read-only. The TabletServerStatus for each server may be null if the tablet
+ * server has not yet responded to a recent request for status.
+ * @param unassigned
+ * A map from unassigned tablet to the last known tablet server. Read-only.
+ * @param assignments
+ * A map from tablet to assigned server. Write-only.
+ */
+ abstract public void getAssignments(SortedMap<TServerInstance,TabletServerStatus> current, Map<KeyExtent,TServerInstance> unassigned,
+ Map<KeyExtent,TServerInstance> assignments);
+
+ /**
+ * Ask the balancer if any migrations are necessary.
+ *
+ * @param current
+ * The current table-summary state of all the online tablet servers. Read-only.
+ * @param migrations
+ * the current set of migrations. Read-only.
+ * @param migrationsOut
+ * new migrations to perform; should not contain tablets in the current set of migrations. Write-only.
+ * @return the time, in milliseconds, to wait before re-balancing.
+ *
+ * This method will not be called when there are unassigned tablets.
+ */
+ public abstract long balance(SortedMap<TServerInstance,TabletServerStatus> current, Set<KeyExtent> migrations, List<TabletMigration> migrationsOut);
+
+ /**
+ * Fetch the tablets for the given table by asking the tablet server. Useful if your balance strategy needs details at the tablet level to decide what tablets
+ * to move.
+ *
+ * @param tserver
+ * The tablet server to ask.
+ * @param tableId
+ * The table id
+ * @return a list of tablet statistics
+ * @throws ThriftSecurityException
+ * tablet server disapproves of your internal System password.
+ * @throws TException
+ * any other problem
+ */
+ public List<TabletStats> getOnlineTabletsForTable(TServerInstance tserver, String tableId) throws ThriftSecurityException, TException {
+ log.debug("Scanning tablet server " + tserver + " for table " + tableId);
+ Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), tserver.getLocation(), configuration.getConfiguration());
+ try {
+ List<TabletStats> onlineTabletsForTable = client.getTabletStats(Tracer.traceInfo(), SecurityConstants.getSystemCredentials(), tableId);
+ return onlineTabletsForTable;
+ } catch (TTransportException e) {
+ log.error("Unable to connect to " + tserver + ": " + e);
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ return null;
+ }
+
+ /**
+ * Utility to ensure that the migrations from balance() are consistent:
+ * <ul>
+ * <li>Tablet objects are not null
+ * <li>Source and destination tablet servers are not null and current
+ * </ul>
+ *
- * @param current
- * @param migrations
+ * @return A list of TabletMigration object that passed sanity checks.
+ */
+ public static List<TabletMigration> checkMigrationSanity(Set<TServerInstance> current, List<TabletMigration> migrations) {
+ List<TabletMigration> result = new ArrayList<TabletMigration>(migrations.size());
+ for (TabletMigration m : migrations) {
+ if (m.tablet == null) {
+ log.warn("Balancer gave back a null tablet " + m);
+ continue;
+ }
+ if (m.newServer == null) {
+ log.warn("Balancer did not set the destination " + m);
+ continue;
+ }
+ if (m.oldServer == null) {
+ log.warn("Balancer did not set the source " + m);
+ continue;
+ }
+ if (!current.contains(m.oldServer)) {
+ log.warn("Balancer wants to move a tablet from a server that is not current: " + m);
+ continue;
+ }
+ if (!current.contains(m.newServer)) {
+ log.warn("Balancer wants to move a tablet to a server that is not current: " + m);
+ continue;
+ }
+ result.add(m);
+ }
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
index f9f03bd,0000000..540ebc0
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
@@@ -1,87 -1,0 +1,81 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.state;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+
+/**
+ * Interface for storing information about tablet assignments. There are three implementations:
+ *
+ * ZooTabletStateStore: information about the root tablet is stored in ZooKeeper MetaDataStateStore: information about the other tablets are stored in the
+ * metadata table
+ *
+ */
+public abstract class TabletStateStore implements Iterable<TabletLocationState> {
+
+ /**
+ * Identifying name for this tablet state store.
+ */
+ abstract public String name();
+
+ /**
+ * Scan the information about the tablets covered by this store
+ */
++ @Override
+ abstract public Iterator<TabletLocationState> iterator();
+
+ /**
+ * Store the assigned locations in the data store.
- *
- * @param assignments
- * @throws DistributedStoreException
+ */
+ abstract public void setFutureLocations(Collection<Assignment> assignments) throws DistributedStoreException;
+
+ /**
+ * Tablet servers will update the data store with the location when they bring the tablet online
- *
- * @param assignments
- * @throws DistributedStoreException
+ */
+ abstract public void setLocations(Collection<Assignment> assignments) throws DistributedStoreException;
+
+ /**
+ * Mark the tablets as having no known or future location.
+ *
+ * @param tablets
+ * the tablets' current information
- * @throws DistributedStoreException
+ */
+ abstract public void unassign(Collection<TabletLocationState> tablets) throws DistributedStoreException;
+
+ public static void unassign(TabletLocationState tls) throws DistributedStoreException {
+ TabletStateStore store;
+ if (tls.extent.isRootTablet()) {
+ store = new ZooTabletStateStore();
+ } else {
+ store = new MetaDataStateStore();
+ }
+ store.unassign(Collections.singletonList(tls));
+ }
+
+ public static void setLocation(Assignment assignment) throws DistributedStoreException {
+ TabletStateStore store;
+ if (assignment.tablet.isRootTablet()) {
+ store = new ZooTabletStateStore();
+ } else {
+ store = new MetaDataStateStore();
+ }
+ store.setLocations(Collections.singletonList(assignment));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/master/tableOps/TraceRepo.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/master/tableOps/TraceRepo.java
index 58a337f,0000000..45f6a60
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/master/tableOps/TraceRepo.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/tableOps/TraceRepo.java
@@@ -1,109 -1,0 +1,84 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.tableOps;
+
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.instrument.Trace;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.accumulo.trace.thrift.TInfo;
+import org.apache.accumulo.fate.Repo;
+
+
+/**
+ *
+ */
+public class TraceRepo<T> implements Repo<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ TInfo tinfo;
+ Repo<T> repo;
+
+ public TraceRepo(Repo<T> repo) {
+ this.repo = repo;
+ tinfo = Tracer.traceInfo();
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.server.fate.Repo#isReady(long, java.lang.Object)
- */
+ @Override
+ public long isReady(long tid, T environment) throws Exception {
+ Span span = Trace.trace(tinfo, repo.getDescription());
+ try {
+ return repo.isReady(tid, environment);
+ } finally {
+ span.stop();
+ }
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.server.fate.Repo#call(long, java.lang.Object)
- */
+ @Override
+ public Repo<T> call(long tid, T environment) throws Exception {
+ Span span = Trace.trace(tinfo, repo.getDescription());
+ try {
+ Repo<T> result = repo.call(tid, environment);
+ if (result == null)
+ return result;
+ return new TraceRepo<T>(result);
+ } finally {
+ span.stop();
+ }
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.server.fate.Repo#undo(long, java.lang.Object)
- */
+ @Override
+ public void undo(long tid, T environment) throws Exception {
+ Span span = Trace.trace(tinfo, repo.getDescription());
+ try {
+ repo.undo(tid, environment);
+ } finally {
+ span.stop();
+ }
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.server.fate.Repo#getDescription()
- */
+ @Override
+ public String getDescription() {
+ return repo.getDescription();
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.server.fate.Repo#getReturn()
- */
+ @Override
+ public String getReturn() {
+ return repo.getReturn();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/metanalysis/LogFileOutputFormat.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/metanalysis/LogFileOutputFormat.java
index 829d7bc,0000000..7e50754
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/metanalysis/LogFileOutputFormat.java
+++ b/server/src/main/java/org/apache/accumulo/server/metanalysis/LogFileOutputFormat.java
@@@ -1,70 -1,0 +1,66 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.metanalysis;
+
+import java.io.IOException;
+
+import org.apache.accumulo.server.logger.LogFileKey;
+import org.apache.accumulo.server.logger.LogFileValue;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+
+/**
+ * Output format for Accumulo write ahead logs.
+ */
+public class LogFileOutputFormat extends FileOutputFormat<LogFileKey,LogFileValue> {
+
+ private static class LogFileRecordWriter extends RecordWriter<LogFileKey,LogFileValue> {
+
+ private FSDataOutputStream out;
+
- /**
- * @param outputPath
- * @throws IOException
- */
+ public LogFileRecordWriter(Path outputPath) throws IOException {
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+
+ out = fs.create(outputPath);
+ }
+
+ @Override
+ public void close(TaskAttemptContext arg0) throws IOException, InterruptedException {
+ out.close();
+ }
+
+ @Override
+ public void write(LogFileKey key, LogFileValue val) throws IOException, InterruptedException {
+ key.write(out);
+ val.write(out);
+ }
+
+ }
+
+ @Override
+ public RecordWriter<LogFileKey,LogFileValue> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
+ Path outputPath = getDefaultWorkFile(context, "");
+ return new LogFileRecordWriter(outputPath);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/metanalysis/PrintEvents.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/metanalysis/PrintEvents.java
index 88f5cbe,0000000..0478d83
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/metanalysis/PrintEvents.java
+++ b/server/src/main/java/org/apache/accumulo/server/metanalysis/PrintEvents.java
@@@ -1,109 -1,0 +1,106 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.metanalysis;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.Constants;
- import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
++import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.server.logger.LogFileValue;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Looks up and prints mutations indexed by IndexMeta
+ */
+public class PrintEvents {
+
+ static class Opts extends ClientOpts {
+ @Parameter(names={"-t", "--tableId"}, description="table id", required=true)
+ String tableId;
+ @Parameter(names={"-e", "--endRow"}, description="end row")
+ String endRow;
+ @Parameter(names={"-t", "--time"}, description="time, in milliseconds", required=true)
+ long time;
+ }
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(PrintEvents.class.getName(), args);
+
+ Connector conn = opts.getConnector();
+
+ printEvents(conn, opts.tableId, opts.endRow, opts.time);
+ }
+
+ /**
+ * @param conn
+ * @param tablePrefix
+ * @param tableId
+ * @param endRow
+ * @param time
+ */
+ private static void printEvents(Connector conn, String tableId, String endRow, Long time) throws Exception {
+ Scanner scanner = conn.createScanner("tabletEvents", new Authorizations());
+ String metaRow = tableId + (endRow == null ? "<" : ";" + endRow);
+ scanner.setRange(new Range(new Key(metaRow, String.format("%020d", time)), true, new Key(metaRow).followingKey(PartialKey.ROW), false));
+ int count = 0;
+
+ String lastLog = null;
+
+ loop1: for (Entry<Key,Value> entry : scanner) {
+ if (entry.getKey().getColumnQualifier().toString().equals("log")) {
+ if (lastLog == null || !lastLog.equals(entry.getValue().toString()))
+ System.out.println("Log : " + entry.getValue());
+ lastLog = entry.getValue().toString();
+ } else if (entry.getKey().getColumnQualifier().toString().equals("mut")) {
+ DataInputStream dis = new DataInputStream(new ByteArrayInputStream(entry.getValue().get()));
+ Mutation m = new Mutation();
+ m.readFields(dis);
+
+ LogFileValue lfv = new LogFileValue();
+ lfv.mutations = Collections.singletonList(m);
+
+ System.out.println(LogFileValue.format(lfv, 1));
+
+ List<ColumnUpdate> columnsUpdates = m.getUpdates();
+ for (ColumnUpdate cu : columnsUpdates) {
+ if (Constants.METADATA_PREV_ROW_COLUMN.equals(new Text(cu.getColumnFamily()), new Text(cu.getColumnQualifier())) && count > 0) {
+ System.out.println("Saw change to prevrow, stopping printing events.");
+ break loop1;
+ }
+ }
+ count++;
+ }
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
index 5a8ddec,0000000..d76c7a3
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
+++ b/server/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
@@@ -1,277 -1,0 +1,273 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.metrics;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.lang.management.ManagementFactory;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+import javax.management.StandardMBean;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.commons.lang.builder.ToStringBuilder;
+import org.apache.commons.lang.time.DateUtils;
+
+public abstract class AbstractMetricsImpl {
+
+ public class Metric {
+
+ private long count = 0;
+ private long avg = 0;
+ private long min = 0;
+ private long max = 0;
+
+ public long getCount() {
+ return count;
+ }
+
+ public long getAvg() {
+ return avg;
+ }
+
+ public long getMin() {
+ return min;
+ }
+
+ public long getMax() {
+ return max;
+ }
+
+ public void incCount() {
+ count++;
+ }
+
+ public void addAvg(long a) {
+ if (a < 0)
+ return;
+ avg = (long) ((avg * .8) + (a * .2));
+ }
+
+ public void addMin(long a) {
+ if (a < 0)
+ return;
+ min = Math.min(min, a);
+ }
+
+ public void addMax(long a) {
+ if (a < 0)
+ return;
+ max = Math.max(max, a);
+ }
+
+ @Override
+ public String toString() {
+ return new ToStringBuilder(this).append("count", count).append("average", avg).append("minimum", min).append("maximum", max).toString();
+ }
+
+ }
+
+ static final org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger(AbstractMetricsImpl.class);
+
+ private static ConcurrentHashMap<String,Metric> registry = new ConcurrentHashMap<String,Metric>();
+
+ private boolean currentlyLogging = false;
+
+ private File logDir = null;
+
+ private String metricsPrefix = null;
+
+ private Date today = new Date();
+
+ private File logFile = null;
+
+ private Writer logWriter = null;
+
+ private SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd");
+
+ private SimpleDateFormat logFormatter = new SimpleDateFormat("yyyyMMddhhmmssz");
+
+ private MetricsConfiguration config = null;
+
+ public AbstractMetricsImpl() {
+ this.metricsPrefix = getMetricsPrefix();
+ config = new MetricsConfiguration(metricsPrefix);
+ }
+
+ /**
+ * Registers a StandardMBean with the MBean Server
- *
- * @throws Exception
+ */
+ public void register(StandardMBean mbean) throws Exception {
+ // Register this object with the MBeanServer
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ if (null == getObjectName())
+ throw new IllegalArgumentException("MBean object name must be set.");
+ mbs.registerMBean(mbean, getObjectName());
+
+ setupLogging();
+ }
+
+ /**
+ * Registers this MBean with the MBean Server
- *
- * @throws Exception
+ */
+ public void register() throws Exception {
+ // Register this object with the MBeanServer
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ if (null == getObjectName())
+ throw new IllegalArgumentException("MBean object name must be set.");
+ mbs.registerMBean(this, getObjectName());
+
+ setupLogging();
+ }
+
+ public void createMetric(String name) {
+ registry.put(name, new Metric());
+ }
+
+ public Metric getMetric(String name) {
+ return registry.get(name);
+ }
+
+ public long getMetricCount(String name) {
+ return registry.get(name).getCount();
+ }
+
+ public long getMetricAvg(String name) {
+ return registry.get(name).getAvg();
+ }
+
+ public long getMetricMin(String name) {
+ return registry.get(name).getMin();
+ }
+
+ public long getMetricMax(String name) {
+ return registry.get(name).getMax();
+ }
+
+ private void setupLogging() throws IOException {
+ if (null == config.getMetricsConfiguration())
+ return;
+ // If we are already logging, then return
+ if (!currentlyLogging && config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ // Check to see if directory exists, else make it
+ String mDir = config.getMetricsConfiguration().getString("logging.dir");
+ if (null != mDir) {
+ File dir = new File(mDir);
+ if (!dir.isDirectory())
+ if (!dir.mkdir())
+ log.warn("Could not create log directory: " + dir);
+ logDir = dir;
+ // Create new log file
+ startNewLog();
+ }
+ currentlyLogging = true;
+ }
+ }
+
+ private void startNewLog() throws IOException {
+ if (null != logWriter) {
+ logWriter.flush();
+ logWriter.close();
+ }
+ logFile = new File(logDir, metricsPrefix + "-" + formatter.format(today) + ".log");
+ if (!logFile.exists()) {
+ if (!logFile.createNewFile()) {
+ log.error("Unable to create new log file");
+ currentlyLogging = false;
+ return;
+ }
+ }
+ logWriter = new OutputStreamWriter(new FileOutputStream(logFile, true), Constants.UTF8);
+ }
+
+ private void writeToLog(String name) throws IOException {
+ if (null == logWriter)
+ return;
+ // Increment the date if we have to
+ Date now = new Date();
+ if (!DateUtils.isSameDay(today, now)) {
+ today = now;
+ startNewLog();
+ }
+ logWriter.append(logFormatter.format(now)).append(" Metric: ").append(name).append(": ").append(registry.get(name).toString()).append("\n");
+ }
+
+ public void add(String name, long time) {
+ if (isEnabled()) {
+ registry.get(name).incCount();
+ registry.get(name).addAvg(time);
+ registry.get(name).addMin(time);
+ registry.get(name).addMax(time);
+ // If we are not currently logging and should be, then initialize
+ if (!currentlyLogging && config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ try {
+ setupLogging();
+ } catch (IOException ioe) {
+ log.error("Error setting up log", ioe);
+ }
+ } else if (currentlyLogging && !config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ // if we are currently logging and shouldn't be, then close logs
+ try {
+ logWriter.flush();
+ logWriter.close();
+ logWriter = null;
+ logFile = null;
+ } catch (Exception e) {
+ log.error("Error stopping metrics logging", e);
+ }
+ currentlyLogging = false;
+ }
+ if (currentlyLogging) {
+ try {
+ writeToLog(name);
+ } catch (IOException ioe) {
+ log.error("Error writing to metrics log", ioe);
+ }
+ }
+ }
+ }
+
+ public boolean isEnabled() {
+ return config.isEnabled();
+ }
+
+ protected abstract ObjectName getObjectName();
+
+ protected abstract String getMetricsPrefix();
+
+ @Override
+ protected void finalize() {
+ if (null != logWriter) {
+ try {
+ logWriter.close();
+ } catch (Exception e) {
+ // do nothing
+ } finally {
+ logWriter = null;
+ }
+ }
+ logFile = null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/security/handler/InsecurePermHandler.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/security/handler/InsecurePermHandler.java
index d51f3f9,0000000..3bda9d6
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/security/handler/InsecurePermHandler.java
+++ b/server/src/main/java/org/apache/accumulo/server/security/handler/InsecurePermHandler.java
@@@ -1,146 -1,0 +1,104 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security.handler;
+
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+
+/**
+ * This is a Permission Handler implementation that doesn't actually do any security. Use at your own risk.
+ */
+public class InsecurePermHandler implements PermissionHandler {
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#initialize(java.lang.String)
- */
+ @Override
+ public void initialize(String instanceId, boolean initialize) {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#validSecurityHandlers(org.apache.accumulo.server.security.handler.Authenticator, org.apache.accumulo.server.security.handler.Authorizor)
- */
+ @Override
+ public boolean validSecurityHandlers(Authenticator authent, Authorizor author) {
+ return true;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#initializeSecurity(java.lang.String)
- */
+ @Override
+ public void initializeSecurity(TCredentials token, String rootuser) throws AccumuloSecurityException {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#hasSystemPermission(java.lang.String, org.apache.accumulo.core.security.SystemPermission)
- */
+ @Override
+ public boolean hasSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ return true;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#hasCachedSystemPermission(java.lang.String, org.apache.accumulo.core.security.SystemPermission)
- */
+ @Override
+ public boolean hasCachedSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ return true;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#hasTablePermission(java.lang.String, java.lang.String, org.apache.accumulo.core.security.TablePermission)
- */
+ @Override
+ public boolean hasTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ return true;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#hasCachedTablePermission(java.lang.String, java.lang.String, org.apache.accumulo.core.security.TablePermission)
- */
+ @Override
+ public boolean hasCachedTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ return true;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#grantSystemPermission(java.lang.String, org.apache.accumulo.core.security.SystemPermission)
- */
+ @Override
+ public void grantSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#revokeSystemPermission(java.lang.String, org.apache.accumulo.core.security.SystemPermission)
- */
+ @Override
+ public void revokeSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#grantTablePermission(java.lang.String, java.lang.String, org.apache.accumulo.core.security.TablePermission)
- */
+ @Override
+ public void grantTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#revokeTablePermission(java.lang.String, java.lang.String, org.apache.accumulo.core.security.TablePermission)
- */
+ @Override
+ public void revokeTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#cleanTablePermissions(java.lang.String)
- */
+ @Override
+ public void cleanTablePermissions(String table) throws AccumuloSecurityException, TableNotFoundException {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#initUser(java.lang.String)
- */
+ @Override
+ public void initUser(String user) throws AccumuloSecurityException {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#dropUser(java.lang.String)
- */
+ @Override
+ public void cleanUser(String user) throws AccumuloSecurityException {
+ return;
+ }
+
+ @Override
+ public void initTable(String table) throws AccumuloSecurityException {
+ }
+
+}
[13/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
index 9c02219,0000000..1f83541
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
@@@ -1,681 -1,0 +1,680 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.mapreduce.lib.util.OutputConfigurator;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.SecurityErrorCode;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * This class allows MapReduce jobs to use Accumulo as the sink for data. This {@link OutputFormat} accepts keys and values of type {@link Text} (for a table
+ * name) and {@link Mutation} from the Map and Reduce functions.
+ *
+ * The user must specify the following via static configurator methods:
+ *
+ * <ul>
+ * <li>{@link AccumuloOutputFormat#setConnectorInfo(Job, String, AuthenticationToken)}
+ * <li>{@link AccumuloOutputFormat#setZooKeeperInstance(Job, String, String)} OR {@link AccumuloOutputFormat#setMockInstance(Job, String)}
+ * </ul>
+ *
+ * Other static methods are optional.
+ */
+public class AccumuloOutputFormat extends OutputFormat<Text,Mutation> {
+
+ private static final Class<?> CLASS = AccumuloOutputFormat.class;
+ protected static final Logger log = Logger.getLogger(CLASS);
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
+ * conversion to a string, and is not intended to be secure.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission if {@link #setCreateTables(Job, boolean)} is set to true)
+ * @param token
+ * the user's password
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public static void setConnectorInfo(Job job, String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ OutputConfigurator.setConnectorInfo(CLASS, job.getConfiguration(), principal, token);
+ }
+
+ /**
+ * Determines if the connector has been configured.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the connector has been configured, false otherwise
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static Boolean isConnectorInfoSet(JobContext context) {
+ return OutputConfigurator.isConnectorInfoSet(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static String getPrincipal(JobContext context) {
+ return OutputConfigurator.getPrincipal(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Gets the serialized token class name from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static String getTokenClass(JobContext context) {
+ return OutputConfigurator.getTokenClass(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Gets the password from the configuration. WARNING: The password is stored in the Configuration and shared with all MapReduce tasks; It is BASE64 encoded to
+ * provide a charset safe conversion to a string, and is not intended to be secure.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the decoded user password
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static byte[] getToken(JobContext context) {
+ return OutputConfigurator.getToken(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Configures a {@link ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @param zooKeepers
+ * a comma-separated list of zookeeper servers
+ * @since 1.5.0
+ */
+ public static void setZooKeeperInstance(Job job, String instanceName, String zooKeepers) {
+ OutputConfigurator.setZooKeeperInstance(CLASS, job.getConfiguration(), instanceName, zooKeepers);
+ }
+
+ /**
+ * Configures a {@link MockInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @since 1.5.0
+ */
+ public static void setMockInstance(Job job, String instanceName) {
+ OutputConfigurator.setMockInstance(CLASS, job.getConfiguration(), instanceName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link Instance} based on the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return an Accumulo instance
+ * @since 1.5.0
+ * @see #setZooKeeperInstance(Job, String, String)
+ * @see #setMockInstance(Job, String)
+ */
+ protected static Instance getInstance(JobContext context) {
+ return OutputConfigurator.getInstance(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param level
+ * the logging level
+ * @since 1.5.0
+ */
+ public static void setLogLevel(Job job, Level level) {
+ OutputConfigurator.setLogLevel(CLASS, job.getConfiguration(), level);
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the log level
+ * @since 1.5.0
+ * @see #setLogLevel(Job, Level)
+ */
+ protected static Level getLogLevel(JobContext context) {
+ return OutputConfigurator.getLogLevel(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Sets the default table name to use if one emits a null in place of a table name for a given mutation. Table names can only be alpha-numeric and
+ * underscores.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.5.0
+ */
+ public static void setDefaultTableName(Job job, String tableName) {
+ OutputConfigurator.setDefaultTableName(CLASS, job.getConfiguration(), tableName);
+ }
+
+ /**
+ * Gets the default table name from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the default table name
+ * @since 1.5.0
+ * @see #setDefaultTableName(Job, String)
+ */
+ protected static String getDefaultTableName(JobContext context) {
+ return OutputConfigurator.getDefaultTableName(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Sets the configuration for for the job's {@link BatchWriter} instances. If not set, a new {@link BatchWriterConfig}, with sensible built-in defaults is
+ * used. Setting the configuration multiple times overwrites any previous configuration.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param bwConfig
+ * the configuration for the {@link BatchWriter}
+ * @since 1.5.0
+ */
+ public static void setBatchWriterOptions(Job job, BatchWriterConfig bwConfig) {
+ OutputConfigurator.setBatchWriterOptions(CLASS, job.getConfiguration(), bwConfig);
+ }
+
+ /**
+ * Gets the {@link BatchWriterConfig} settings.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the configuration object
+ * @since 1.5.0
+ * @see #setBatchWriterOptions(Job, BatchWriterConfig)
+ */
+ protected static BatchWriterConfig getBatchWriterOptions(JobContext context) {
+ return OutputConfigurator.getBatchWriterOptions(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Sets the directive to create new tables, as necessary. Table names can only be alpha-numeric and underscores.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setCreateTables(Job job, boolean enableFeature) {
+ OutputConfigurator.setCreateTables(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether tables are permitted to be created as needed.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the feature is disabled, false otherwise
+ * @since 1.5.0
+ * @see #setCreateTables(Job, boolean)
+ */
+ protected static Boolean canCreateTables(JobContext context) {
+ return OutputConfigurator.canCreateTables(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Sets the directive to use simulation mode for this job. In simulation mode, no output is produced. This is useful for testing.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setSimulationMode(Job job, boolean enableFeature) {
+ OutputConfigurator.setSimulationMode(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether this feature is enabled.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setSimulationMode(Job, boolean)
+ */
+ protected static Boolean getSimulationMode(JobContext context) {
+ return OutputConfigurator.getSimulationMode(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * A base class to be used to create {@link RecordWriter} instances that write to Accumulo.
+ */
+ protected static class AccumuloRecordWriter extends RecordWriter<Text,Mutation> {
+ private MultiTableBatchWriter mtbw = null;
+ private HashMap<Text,BatchWriter> bws = null;
+ private Text defaultTableName = null;
+
+ private boolean simulate = false;
+ private boolean createTables = false;
+
+ private long mutCount = 0;
+ private long valCount = 0;
+
+ private Connector conn;
+
+ protected AccumuloRecordWriter(TaskAttemptContext context) throws AccumuloException, AccumuloSecurityException, IOException {
+ Level l = getLogLevel(context);
+ if (l != null)
+ log.setLevel(getLogLevel(context));
+ this.simulate = getSimulationMode(context);
+ this.createTables = canCreateTables(context);
+
+ if (simulate)
+ log.info("Simulating output only. No writes to tables will occur");
+
+ this.bws = new HashMap<Text,BatchWriter>();
+
+ String tname = getDefaultTableName(context);
+ this.defaultTableName = (tname == null) ? null : new Text(tname);
+
+ if (!simulate) {
+ this.conn = getInstance(context).getConnector(getPrincipal(context), CredentialHelper.extractToken(getTokenClass(context), getToken(context)));
+ mtbw = conn.createMultiTableBatchWriter(getBatchWriterOptions(context));
+ }
+ }
+
+ /**
+ * Push a mutation into a table. If table is null, the defaultTable will be used. If canCreateTable is set, the table will be created if it does not exist.
+ * The table name must only contain alphanumerics and underscore.
+ */
+ @Override
+ public void write(Text table, Mutation mutation) throws IOException {
+ if (table == null || table.toString().isEmpty())
+ table = this.defaultTableName;
+
+ if (!simulate && table == null)
+ throw new IOException("No table or default table specified. Try simulation mode next time");
+
+ ++mutCount;
+ valCount += mutation.size();
+ printMutation(table, mutation);
+
+ if (simulate)
+ return;
+
+ if (!bws.containsKey(table))
+ try {
+ addTable(table);
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new IOException(e);
+ }
+
+ try {
+ bws.get(table).addMutation(mutation);
+ } catch (MutationsRejectedException e) {
+ throw new IOException(e);
+ }
+ }
+
+ public void addTable(Text tableName) throws AccumuloException, AccumuloSecurityException {
+ if (simulate) {
+ log.info("Simulating adding table: " + tableName);
+ return;
+ }
+
+ log.debug("Adding table: " + tableName);
+ BatchWriter bw = null;
+ String table = tableName.toString();
+
+ if (createTables && !conn.tableOperations().exists(table)) {
+ try {
+ conn.tableOperations().create(table);
+ } catch (AccumuloSecurityException e) {
+ log.error("Accumulo security violation creating " + table, e);
+ throw e;
+ } catch (TableExistsException e) {
+ // Shouldn't happen
+ }
+ }
+
+ try {
+ bw = mtbw.getBatchWriter(table);
+ } catch (TableNotFoundException e) {
+ log.error("Accumulo table " + table + " doesn't exist and cannot be created.", e);
+ throw new AccumuloException(e);
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (AccumuloSecurityException e) {
+ throw e;
+ }
+
+ if (bw != null)
+ bws.put(tableName, bw);
+ }
+
+ private int printMutation(Text table, Mutation m) {
+ if (log.isTraceEnabled()) {
+ log.trace(String.format("Table %s row key: %s", table, hexDump(m.getRow())));
+ for (ColumnUpdate cu : m.getUpdates()) {
+ log.trace(String.format("Table %s column: %s:%s", table, hexDump(cu.getColumnFamily()), hexDump(cu.getColumnQualifier())));
+ log.trace(String.format("Table %s security: %s", table, new ColumnVisibility(cu.getColumnVisibility()).toString()));
+ log.trace(String.format("Table %s value: %s", table, hexDump(cu.getValue())));
+ }
+ }
+ return m.getUpdates().size();
+ }
+
+ private String hexDump(byte[] ba) {
+ StringBuilder sb = new StringBuilder();
+ for (byte b : ba) {
+ if ((b > 0x20) && (b < 0x7e))
+ sb.append((char) b);
+ else
+ sb.append(String.format("x%02x", b));
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public void close(TaskAttemptContext attempt) throws IOException, InterruptedException {
+ log.debug("mutations written: " + mutCount + ", values written: " + valCount);
+ if (simulate)
+ return;
+
+ try {
+ mtbw.close();
+ } catch (MutationsRejectedException e) {
+ if (e.getAuthorizationFailuresMap().size() >= 0) {
+ HashMap<String,Set<SecurityErrorCode>> tables = new HashMap<String,Set<SecurityErrorCode>>();
+ for (Entry<KeyExtent,Set<SecurityErrorCode>> ke : e.getAuthorizationFailuresMap().entrySet()) {
+ Set<SecurityErrorCode> secCodes = tables.get(ke.getKey().getTableId().toString());
+ if (secCodes == null) {
+ secCodes = new HashSet<SecurityErrorCode>();
+ tables.put(ke.getKey().getTableId().toString(), secCodes);
+ }
+ secCodes.addAll(ke.getValue());
+ }
+
+ log.error("Not authorized to write to tables : " + tables);
+ }
+
+ if (e.getConstraintViolationSummaries().size() > 0) {
+ log.error("Constraint violations : " + e.getConstraintViolationSummaries().size());
+ }
+ }
+ }
+ }
+
+ @Override
+ public void checkOutputSpecs(JobContext job) throws IOException {
+ if (!isConnectorInfoSet(job))
+ throw new IOException("Connector info has not been set.");
+ try {
+ // if the instance isn't configured, it will complain here
+ Connector c = getInstance(job).getConnector(getPrincipal(job), CredentialHelper.extractToken(getTokenClass(job), getToken(job)));
+ if (!c.securityOperations().authenticateUser(getPrincipal(job), CredentialHelper.extractToken(getTokenClass(job), getToken(job))))
+ throw new IOException("Unable to authenticate user");
+ } catch (AccumuloException e) {
+ throw new IOException(e);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public OutputCommitter getOutputCommitter(TaskAttemptContext context) {
+ return new NullOutputFormat<Text,Mutation>().getOutputCommitter(context);
+ }
+
+ @Override
+ public RecordWriter<Text,Mutation> getRecordWriter(TaskAttemptContext attempt) throws IOException {
+ try {
+ return new AccumuloRecordWriter(attempt);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ // ----------------------------------------------------------------------------------------------------
+ // Everything below this line is deprecated and should go away in future versions
+ // ----------------------------------------------------------------------------------------------------
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setConnectorInfo(Job, String, AuthenticationToken)}, {@link #setCreateTables(Job, boolean)}, and
+ * {@link #setDefaultTableName(Job, String)} instead.
+ */
+ @Deprecated
+ public static void setOutputInfo(Configuration conf, String user, byte[] passwd, boolean createTables, String defaultTable) {
+ try {
+ OutputConfigurator.setConnectorInfo(CLASS, conf, user, new PasswordToken(passwd));
+ } catch (AccumuloSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ OutputConfigurator.setCreateTables(CLASS, conf, createTables);
+ OutputConfigurator.setDefaultTableName(CLASS, conf, defaultTable);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setZooKeeperInstance(Job, String, String)} instead.
+ */
+ @Deprecated
+ public static void setZooKeeperInstance(Configuration conf, String instanceName, String zooKeepers) {
+ OutputConfigurator.setZooKeeperInstance(CLASS, conf, instanceName, zooKeepers);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setMockInstance(Job, String)} instead.
+ */
+ @Deprecated
+ public static void setMockInstance(Configuration conf, String instanceName) {
+ OutputConfigurator.setMockInstance(CLASS, conf, instanceName);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setBatchWriterOptions(Job, BatchWriterConfig)} instead.
+ */
+ @Deprecated
+ public static void setMaxMutationBufferSize(Configuration conf, long numberOfBytes) {
+ BatchWriterConfig bwConfig = OutputConfigurator.getBatchWriterOptions(CLASS, conf);
+ bwConfig.setMaxMemory(numberOfBytes);
+ OutputConfigurator.setBatchWriterOptions(CLASS, conf, bwConfig);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setBatchWriterOptions(Job, BatchWriterConfig)} instead.
+ */
+ @Deprecated
+ public static void setMaxLatency(Configuration conf, int numberOfMilliseconds) {
+ BatchWriterConfig bwConfig = OutputConfigurator.getBatchWriterOptions(CLASS, conf);
+ bwConfig.setMaxLatency(numberOfMilliseconds, TimeUnit.MILLISECONDS);
+ OutputConfigurator.setBatchWriterOptions(CLASS, conf, bwConfig);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setBatchWriterOptions(Job, BatchWriterConfig)} instead.
+ */
+ @Deprecated
+ public static void setMaxWriteThreads(Configuration conf, int numberOfThreads) {
+ BatchWriterConfig bwConfig = OutputConfigurator.getBatchWriterOptions(CLASS, conf);
+ bwConfig.setMaxWriteThreads(numberOfThreads);
+ OutputConfigurator.setBatchWriterOptions(CLASS, conf, bwConfig);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setLogLevel(Job, Level)} instead.
+ */
+ @Deprecated
+ public static void setLogLevel(Configuration conf, Level level) {
+ OutputConfigurator.setLogLevel(CLASS, conf, level);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setSimulationMode(Job, boolean)} instead.
+ */
+ @Deprecated
+ public static void setSimulationMode(Configuration conf) {
+ OutputConfigurator.setSimulationMode(CLASS, conf, true);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getToken(JobContext)} instead.
+ */
+ @Deprecated
+ protected static String getPrincipal(Configuration conf) {
+ return OutputConfigurator.getPrincipal(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getToken(JobContext)} instead.
+ */
+ @Deprecated
+ protected static byte[] getToken(Configuration conf) {
+ return OutputConfigurator.getToken(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #canCreateTables(JobContext)} instead.
+ */
+ @Deprecated
+ protected static boolean canCreateTables(Configuration conf) {
+ return OutputConfigurator.canCreateTables(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getDefaultTableName(JobContext)} instead.
+ */
+ @Deprecated
+ protected static String getDefaultTableName(Configuration conf) {
+ return OutputConfigurator.getDefaultTableName(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getInstance(JobContext)} instead.
+ */
+ @Deprecated
+ protected static Instance getInstance(Configuration conf) {
+ return OutputConfigurator.getInstance(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getBatchWriterOptions(JobContext)} instead.
+ */
+ @Deprecated
+ protected static long getMaxMutationBufferSize(Configuration conf) {
+ return OutputConfigurator.getBatchWriterOptions(CLASS, conf).getMaxMemory();
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getBatchWriterOptions(JobContext)} instead.
+ */
+ @Deprecated
+ protected static int getMaxLatency(Configuration conf) {
+ return (int) OutputConfigurator.getBatchWriterOptions(CLASS, conf).getMaxLatency(TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getBatchWriterOptions(JobContext)} instead.
+ */
+ @Deprecated
+ protected static int getMaxWriteThreads(Configuration conf) {
+ return OutputConfigurator.getBatchWriterOptions(CLASS, conf).getMaxWriteThreads();
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getLogLevel(JobContext)} instead.
+ */
+ @Deprecated
+ protected static Level getLogLevel(Configuration conf) {
+ return OutputConfigurator.getLogLevel(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getSimulationMode(JobContext)} instead.
+ */
+ @Deprecated
+ protected static boolean getSimulationMode(Configuration conf) {
+ return OutputConfigurator.getSimulationMode(CLASS, conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
index 0e9444d,0000000..710c565
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
@@@ -1,1337 -1,0 +1,1336 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Method;
+import java.net.InetAddress;
+import java.net.URLDecoder;
+import java.net.URLEncoder;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.StringTokenizer;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientSideIteratorScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.RowIterator;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableDeletedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.impl.OfflineScanner;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.TabletLocator;
+import org.apache.accumulo.core.client.mapreduce.lib.util.InputConfigurator;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.VersioningIterator;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * This abstract {@link InputFormat} class allows MapReduce jobs to use Accumulo as the source of K,V pairs.
+ * <p>
+ * Subclasses must implement a {@link #createRecordReader(InputSplit, TaskAttemptContext)} to provide a {@link RecordReader} for K,V.
+ * <p>
+ * A static base class, RecordReaderBase, is provided to retrieve Accumulo {@link Key}/{@link Value} pairs, but one must implement its
+ * {@link RecordReaderBase#nextKeyValue()} to transform them to the desired generic types K,V.
+ * <p>
+ * See {@link AccumuloInputFormat} for an example implementation.
+ */
+public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
+
+ private static final Class<?> CLASS = AccumuloInputFormat.class;
+ protected static final Logger log = Logger.getLogger(CLASS);
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
+ * conversion to a string, and is not intended to be secure.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission)
+ * @param token
+ * the user's password
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public static void setConnectorInfo(Job job, String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ InputConfigurator.setConnectorInfo(CLASS, job.getConfiguration(), principal, token);
+ }
+
+ /**
+ * Determines if the connector has been configured.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the connector has been configured, false otherwise
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static Boolean isConnectorInfoSet(JobContext context) {
+ return InputConfigurator.isConnectorInfoSet(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static String getPrincipal(JobContext context) {
+ return InputConfigurator.getPrincipal(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Gets the serialized token class from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static String getTokenClass(JobContext context) {
+ return InputConfigurator.getTokenClass(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Gets the password from the configuration. WARNING: The password is stored in the Configuration and shared with all MapReduce tasks; It is BASE64 encoded to
+ * provide a charset safe conversion to a string, and is not intended to be secure.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the decoded user password
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static byte[] getToken(JobContext context) {
+ return InputConfigurator.getToken(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Configures a {@link ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @param zooKeepers
+ * a comma-separated list of zookeeper servers
+ * @since 1.5.0
+ */
+ public static void setZooKeeperInstance(Job job, String instanceName, String zooKeepers) {
+ InputConfigurator.setZooKeeperInstance(CLASS, job.getConfiguration(), instanceName, zooKeepers);
+ }
+
+ /**
+ * Configures a {@link MockInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @since 1.5.0
+ */
+ public static void setMockInstance(Job job, String instanceName) {
+ InputConfigurator.setMockInstance(CLASS, job.getConfiguration(), instanceName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link Instance} based on the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return an Accumulo instance
+ * @since 1.5.0
+ * @see #setZooKeeperInstance(Job, String, String)
+ * @see #setMockInstance(Job, String)
+ */
+ protected static Instance getInstance(JobContext context) {
+ return InputConfigurator.getInstance(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param level
+ * the logging level
+ * @since 1.5.0
+ */
+ public static void setLogLevel(Job job, Level level) {
+ InputConfigurator.setLogLevel(CLASS, job.getConfiguration(), level);
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the log level
+ * @since 1.5.0
+ * @see #setLogLevel(Job, Level)
+ */
+ protected static Level getLogLevel(JobContext context) {
+ return InputConfigurator.getLogLevel(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Sets the name of the input table, over which this job will scan.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.5.0
+ */
+ public static void setInputTableName(Job job, String tableName) {
+ InputConfigurator.setInputTableName(CLASS, job.getConfiguration(), tableName);
+ }
+
+ /**
+ * Gets the table name from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the table name
+ * @since 1.5.0
+ * @see #setInputTableName(Job, String)
+ */
+ protected static String getInputTableName(JobContext context) {
+ return InputConfigurator.getInputTableName(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Sets the {@link Authorizations} used to scan. Must be a subset of the user's authorization. Defaults to the empty set.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param auths
+ * the user's authorizations
+ * @since 1.5.0
+ */
+ public static void setScanAuthorizations(Job job, Authorizations auths) {
+ InputConfigurator.setScanAuthorizations(CLASS, job.getConfiguration(), auths);
+ }
+
+ /**
+ * Gets the authorizations to set for the scans from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the Accumulo scan authorizations
+ * @since 1.5.0
+ * @see #setScanAuthorizations(Job, Authorizations)
+ */
+ protected static Authorizations getScanAuthorizations(JobContext context) {
+ return InputConfigurator.getScanAuthorizations(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Sets the input ranges to scan for this job. If not set, the entire table will be scanned.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param ranges
+ * the ranges that will be mapped over
+ * @since 1.5.0
+ */
+ public static void setRanges(Job job, Collection<Range> ranges) {
+ InputConfigurator.setRanges(CLASS, job.getConfiguration(), ranges);
+ }
+
+ /**
+ * Gets the ranges to scan over from a job.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the ranges
+ * @throws IOException
+ * if the ranges have been encoded improperly
+ * @since 1.5.0
+ * @see #setRanges(Job, Collection)
+ */
+ protected static List<Range> getRanges(JobContext context) throws IOException {
+ return InputConfigurator.getRanges(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Restricts the columns that will be mapped over for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param columnFamilyColumnQualifierPairs
+ * a pair of {@link Text} objects corresponding to column family and column qualifier. If the column qualifier is null, the entire column family is
+ * selected. An empty set is the default and is equivalent to scanning the all columns.
+ * @since 1.5.0
+ */
+ public static void fetchColumns(Job job, Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
+ InputConfigurator.fetchColumns(CLASS, job.getConfiguration(), columnFamilyColumnQualifierPairs);
+ }
+
+ /**
+ * Gets the columns to be mapped over from this job.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return a set of columns
+ * @since 1.5.0
+ * @see #fetchColumns(Job, Collection)
+ */
+ protected static Set<Pair<Text,Text>> getFetchedColumns(JobContext context) {
+ return InputConfigurator.getFetchedColumns(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Encode an iterator on the input for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param cfg
+ * the configuration of the iterator
+ * @since 1.5.0
+ */
+ public static void addIterator(Job job, IteratorSetting cfg) {
+ InputConfigurator.addIterator(CLASS, job.getConfiguration(), cfg);
+ }
+
+ /**
+ * Gets a list of the iterator settings (for iterators to apply to a scanner) from this configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return a list of iterators
+ * @since 1.5.0
+ * @see #addIterator(Job, IteratorSetting)
+ */
+ protected static List<IteratorSetting> getIterators(JobContext context) {
+ return InputConfigurator.getIterators(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Controls the automatic adjustment of ranges for this job. This feature merges overlapping ranges, then splits them to align with tablet boundaries.
+ * Disabling this feature will cause exactly one Map task to be created for each specified range. The default setting is enabled. *
+ *
+ * <p>
+ * By default, this feature is <b>enabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @see #setRanges(Job, Collection)
+ * @since 1.5.0
+ */
+ public static void setAutoAdjustRanges(Job job, boolean enableFeature) {
+ InputConfigurator.setAutoAdjustRanges(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has auto-adjust ranges enabled.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return false if the feature is disabled, true otherwise
+ * @since 1.5.0
+ * @see #setAutoAdjustRanges(Job, boolean)
+ */
+ protected static boolean getAutoAdjustRanges(JobContext context) {
+ return InputConfigurator.getAutoAdjustRanges(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Controls the use of the {@link IsolatedScanner} in this job.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setScanIsolation(Job job, boolean enableFeature) {
+ InputConfigurator.setScanIsolation(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has isolation enabled.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setScanIsolation(Job, boolean)
+ */
+ protected static boolean isIsolated(JobContext context) {
+ return InputConfigurator.isIsolated(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Controls the use of the {@link ClientSideIteratorScanner} in this job. Enabling this feature will cause the iterator stack to be constructed within the Map
+ * task, rather than within the Accumulo TServer. To use this feature, all classes needed for those iterators must be available on the classpath for the task.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setLocalIterators(Job job, boolean enableFeature) {
+ InputConfigurator.setLocalIterators(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration uses local iterators.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setLocalIterators(Job, boolean)
+ */
+ protected static boolean usesLocalIterators(JobContext context) {
+ return InputConfigurator.usesLocalIterators(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * <p>
+ * Enable reading offline tables. By default, this feature is disabled and only online tables are scanned. This will make the map reduce job directly read the
+ * table's files. If the table is not offline, then the job will fail. If the table comes online during the map reduce job, it is likely that the job will
+ * fail.
+ *
+ * <p>
+ * To use this option, the map reduce user will need access to read the Accumulo directory in HDFS.
+ *
+ * <p>
+ * Reading the offline table will create the scan time iterator stack in the map process. So any iterators that are configured for the table will need to be
+ * on the mapper's classpath.
+ *
+ * <p>
+ * One way to use this feature is to clone a table, take the clone offline, and use the clone as the input table for a map reduce job. If you plan to map
+ * reduce over the data many times, it may be better to the compact the table, clone it, take it offline, and use the clone for all map reduce jobs. The
+ * reason to do this is that compaction will reduce each tablet in the table to one file, and it is faster to read from one file.
+ *
+ * <p>
+ * There are two possible advantages to reading a tables file directly out of HDFS. First, you may see better read performance. Second, it will support
+ * speculative execution better. When reading an online table speculative execution can put more load on an already slow tablet server.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setOfflineTableScan(Job job, boolean enableFeature) {
+ InputConfigurator.setOfflineTableScan(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has the offline table scan feature enabled.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setOfflineTableScan(Job, boolean)
+ */
+ protected static boolean isOfflineScan(JobContext context) {
+ return InputConfigurator.isOfflineScan(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Initializes an Accumulo {@link TabletLocator} based on the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return an Accumulo tablet locator
+ * @throws TableNotFoundException
+ * if the table name set on the configuration doesn't exist
+ * @since 1.5.0
+ */
+ protected static TabletLocator getTabletLocator(JobContext context) throws TableNotFoundException {
+ return InputConfigurator.getTabletLocator(CLASS, getConfiguration(context));
+ }
+
+ // InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext job)
+ /**
+ * Check whether a configuration is fully configured to be used with an Accumulo {@link org.apache.hadoop.mapreduce.InputFormat}.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @throws IOException
+ * if the context is improperly configured
+ * @since 1.5.0
+ */
+ protected static void validateOptions(JobContext context) throws IOException {
+ InputConfigurator.validateOptions(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * An abstract base class to be used to create {@link RecordReader} instances that convert from Accumulo {@link Key}/{@link Value} pairs to the user's K/V
+ * types.
+ *
+ * Subclasses must implement {@link #nextKeyValue()} and use it to update the following variables:
+ * <ul>
+ * <li>K {@link #currentK}</li>
+ * <li>V {@link #currentV}</li>
+ * <li>Key {@link #currentKey} (used for progress reporting)</li>
+ * <li>int {@link #numKeysRead} (used for progress reporting)</li>
+ * </ul>
+ */
+ protected abstract static class RecordReaderBase<K,V> extends RecordReader<K,V> {
+ protected long numKeysRead;
+ protected Iterator<Entry<Key,Value>> scannerIterator;
+ protected org.apache.accumulo.core.client.mapreduce.RangeInputSplit split;
+
+ /**
+ * Apply the configured iterators from the configuration to the scanner.
+ *
+ * @param scanner
+ * the scanner to configure
+ */
+ protected void setupIterators(List<IteratorSetting> iterators, Scanner scanner) {
+ for (IteratorSetting iterator : iterators) {
+ scanner.addScanIterator(iterator);
+ }
+ }
+
+ /**
+ * Initialize a scanner over the given input split using this task attempt configuration.
+ */
+ @Override
+ public void initialize(InputSplit inSplit, TaskAttemptContext attempt) throws IOException {
+ Scanner scanner;
+ split = (org.apache.accumulo.core.client.mapreduce.RangeInputSplit) inSplit;
+ log.debug("Initializing input split: " + split.getRange());
+
+ Instance instance = split.getInstance();
+ if (null == instance) {
+ instance = getInstance(attempt);
+ }
+
+ String principal = split.getPrincipal();
+ if (null == principal) {
+ principal = getPrincipal(attempt);
+ }
+
+ AuthenticationToken token = split.getToken();
+ if (null == token) {
+ String tokenClass = getTokenClass(attempt);
+ byte[] tokenBytes = getToken(attempt);
+ try {
+ token = CredentialHelper.extractToken(tokenClass, tokenBytes);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+ }
+
+ Authorizations authorizations = split.getAuths();
+ if (null == authorizations) {
+ authorizations = getScanAuthorizations(attempt);
+ }
+
+ String table = split.getTable();
+ if (null == table) {
+ table = getInputTableName(attempt);
+ }
+
+ Boolean isOffline = split.isOffline();
+ if (null == isOffline) {
+ isOffline = isOfflineScan(attempt);
+ }
+
+ Boolean isIsolated = split.isIsolatedScan();
+ if (null == isIsolated) {
+ isIsolated = isIsolated(attempt);
+ }
+
+ Boolean usesLocalIterators = split.usesLocalIterators();
+ if (null == usesLocalIterators) {
+ usesLocalIterators = usesLocalIterators(attempt);
+ }
+
+ List<IteratorSetting> iterators = split.getIterators();
+ if (null == iterators) {
+ iterators = getIterators(attempt);
+ }
+
+ Set<Pair<Text,Text>> columns = split.getFetchedColumns();
+ if (null == columns) {
+ columns = getFetchedColumns(attempt);
+ }
+
+ try {
+ log.debug("Creating connector with user: " + principal);
+ Connector conn = instance.getConnector(principal, token);
+ log.debug("Creating scanner for table: " + table);
+ log.debug("Authorizations are: " + authorizations);
+ if (isOffline) {
+ String tokenClass = token.getClass().getCanonicalName();
+ ByteBuffer tokenBuffer = ByteBuffer.wrap(CredentialHelper.toBytes(token));
+ scanner = new OfflineScanner(instance, new TCredentials(principal, tokenClass, tokenBuffer, instance.getInstanceID()), Tables.getTableId(instance,
+ table), authorizations);
+ } else {
+ scanner = conn.createScanner(table, authorizations);
+ }
+ if (isIsolated) {
+ log.info("Creating isolated scanner");
+ scanner = new IsolatedScanner(scanner);
+ }
+ if (usesLocalIterators) {
+ log.info("Using local iterators");
+ scanner = new ClientSideIteratorScanner(scanner);
+ }
+ setupIterators(iterators, scanner);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ // setup a scanner within the bounds of this split
+ for (Pair<Text,Text> c : columns) {
+ if (c.getSecond() != null) {
+ log.debug("Fetching column " + c.getFirst() + ":" + c.getSecond());
+ scanner.fetchColumn(c.getFirst(), c.getSecond());
+ } else {
+ log.debug("Fetching column family " + c.getFirst());
+ scanner.fetchColumnFamily(c.getFirst());
+ }
+ }
+
+ scanner.setRange(split.getRange());
+
+ numKeysRead = 0;
+
+ // do this last after setting all scanner options
+ scannerIterator = scanner.iterator();
+ }
+
+ @Override
+ public void close() {}
+
+ @Override
+ public float getProgress() throws IOException {
+ if (numKeysRead > 0 && currentKey == null)
+ return 1.0f;
+ return split.getProgress(currentKey);
+ }
+
+ protected K currentK = null;
+ protected V currentV = null;
+ protected Key currentKey = null;
+ protected Value currentValue = null;
+
+ @Override
+ public K getCurrentKey() throws IOException, InterruptedException {
+ return currentK;
+ }
+
+ @Override
+ public V getCurrentValue() throws IOException, InterruptedException {
+ return currentV;
+ }
+ }
+
+ Map<String,Map<KeyExtent,List<Range>>> binOfflineTable(JobContext context, String tableName, List<Range> ranges) throws TableNotFoundException,
+ AccumuloException, AccumuloSecurityException {
+
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+
+ Instance instance = getInstance(context);
+ Connector conn = instance.getConnector(getPrincipal(context), CredentialHelper.extractToken(getTokenClass(context), getToken(context)));
+ String tableId = Tables.getTableId(instance, tableName);
+
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ Tables.clearCache(instance);
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ throw new AccumuloException("Table is online " + tableName + "(" + tableId + ") cannot scan table in offline mode ");
+ }
+ }
+
+ for (Range range : ranges) {
+ Text startRow;
+
+ if (range.getStartKey() != null)
+ startRow = range.getStartKey().getRow();
+ else
+ startRow = new Text();
+
+ Range metadataRange = new Range(new KeyExtent(new Text(tableId), startRow, null).getMetadataEntry(), true, null, false);
+ Scanner scanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ Constants.METADATA_PREV_ROW_COLUMN.fetch(scanner);
+ scanner.fetchColumnFamily(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY);
+ scanner.setRange(metadataRange);
+
+ RowIterator rowIter = new RowIterator(scanner);
+
+ KeyExtent lastExtent = null;
+
+ while (rowIter.hasNext()) {
+ Iterator<Entry<Key,Value>> row = rowIter.next();
+ String last = "";
+ KeyExtent extent = null;
+ String location = null;
+
+ while (row.hasNext()) {
+ Entry<Key,Value> entry = row.next();
+ Key key = entry.getKey();
+
+ if (key.getColumnFamily().equals(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY)) {
+ last = entry.getValue().toString();
+ }
+
+ if (key.getColumnFamily().equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY)
+ || key.getColumnFamily().equals(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY)) {
+ location = entry.getValue().toString();
+ }
+
+ if (Constants.METADATA_PREV_ROW_COLUMN.hasColumns(key)) {
+ extent = new KeyExtent(key.getRow(), entry.getValue());
+ }
+
+ }
+
+ if (location != null)
+ return null;
+
+ if (!extent.getTableId().toString().equals(tableId)) {
+ throw new AccumuloException("Saw unexpected table Id " + tableId + " " + extent);
+ }
+
+ if (lastExtent != null && !extent.isPreviousExtent(lastExtent)) {
+ throw new AccumuloException(" " + lastExtent + " is not previous extent " + extent);
+ }
+
+ Map<KeyExtent,List<Range>> tabletRanges = binnedRanges.get(last);
+ if (tabletRanges == null) {
+ tabletRanges = new HashMap<KeyExtent,List<Range>>();
+ binnedRanges.put(last, tabletRanges);
+ }
+
+ List<Range> rangeList = tabletRanges.get(extent);
+ if (rangeList == null) {
+ rangeList = new ArrayList<Range>();
+ tabletRanges.put(extent, rangeList);
+ }
+
+ rangeList.add(range);
+
+ if (extent.getEndRow() == null || range.afterEndKey(new Key(extent.getEndRow()).followingKey(PartialKey.ROW))) {
+ break;
+ }
+
+ lastExtent = extent;
+ }
+
+ }
+
+ return binnedRanges;
+ }
+
+ /**
+ * Read the metadata table to get tablets and match up ranges to them.
+ */
+ @Override
+ public List<InputSplit> getSplits(JobContext context) throws IOException {
+ Level logLevel = getLogLevel(context);
+ log.setLevel(logLevel);
+
+ validateOptions(context);
+
+ String tableName = getInputTableName(context);
+ boolean autoAdjust = getAutoAdjustRanges(context);
+ List<Range> ranges = autoAdjust ? Range.mergeOverlapping(getRanges(context)) : getRanges(context);
+ Instance instance = getInstance(context);
+ boolean offline = isOfflineScan(context);
+ boolean isolated = isIsolated(context);
+ boolean localIterators = usesLocalIterators(context);
+ boolean mockInstance = (null != instance && MockInstance.class.equals(instance.getClass()));
+ Set<Pair<Text,Text>> fetchedColumns = getFetchedColumns(context);
+ Authorizations auths = getScanAuthorizations(context);
+ String principal = getPrincipal(context);
+ String tokenClass = getTokenClass(context);
+ byte[] tokenBytes = getToken(context);
+
+ AuthenticationToken token;
+ try {
+ token = CredentialHelper.extractToken(tokenClass, tokenBytes);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+
+ List<IteratorSetting> iterators = getIterators(context);
+
+ if (ranges.isEmpty()) {
+ ranges = new ArrayList<Range>(1);
+ ranges.add(new Range());
+ }
+
+ // get the metadata information for these ranges
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+ TabletLocator tl;
+ try {
+ if (isOfflineScan(context)) {
+ binnedRanges = binOfflineTable(context, tableName, ranges);
+ while (binnedRanges == null) {
+ // Some tablets were still online, try again
+ UtilWaitThread.sleep(100 + (int) (Math.random() * 100)); // sleep randomly between 100 and 200 ms
+ binnedRanges = binOfflineTable(context, tableName, ranges);
+ }
+ } else {
+ String tableId = null;
+ tl = getTabletLocator(context);
+ // its possible that the cache could contain complete, but old information about a tables tablets... so clear it
+ tl.invalidateCache();
+ while (!tl.binRanges(ranges, binnedRanges, new TCredentials(principal, tokenClass, ByteBuffer.wrap(tokenBytes), instance.getInstanceID())).isEmpty()) {
+ if (!(instance instanceof MockInstance)) {
+ if (tableId == null)
+ tableId = Tables.getTableId(instance, tableName);
+ if (!Tables.exists(instance, tableId))
+ throw new TableDeletedException(tableId);
+ if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
+ throw new TableOfflineException(instance, tableId);
+ }
+ binnedRanges.clear();
+ log.warn("Unable to locate bins for specified ranges. Retrying.");
+ UtilWaitThread.sleep(100 + (int) (Math.random() * 100)); // sleep randomly between 100 and 200 ms
+ tl.invalidateCache();
+ }
+ }
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ ArrayList<InputSplit> splits = new ArrayList<InputSplit>(ranges.size());
+ HashMap<Range,ArrayList<String>> splitsToAdd = null;
+
+ if (!autoAdjust)
+ splitsToAdd = new HashMap<Range,ArrayList<String>>();
+
+ HashMap<String,String> hostNameCache = new HashMap<String,String>();
+
+ for (Entry<String,Map<KeyExtent,List<Range>>> tserverBin : binnedRanges.entrySet()) {
+ String ip = tserverBin.getKey().split(":", 2)[0];
+ String location = hostNameCache.get(ip);
+ if (location == null) {
+ InetAddress inetAddress = InetAddress.getByName(ip);
+ location = inetAddress.getHostName();
+ hostNameCache.put(ip, location);
+ }
+
+ for (Entry<KeyExtent,List<Range>> extentRanges : tserverBin.getValue().entrySet()) {
+ Range ke = extentRanges.getKey().toDataRange();
+ for (Range r : extentRanges.getValue()) {
+ if (autoAdjust) {
+ // divide ranges into smaller ranges, based on the tablets
+ splits.add(new org.apache.accumulo.core.client.mapreduce.RangeInputSplit(ke.clip(r), new String[] {location}));
+ } else {
+ // don't divide ranges
+ ArrayList<String> locations = splitsToAdd.get(r);
+ if (locations == null)
+ locations = new ArrayList<String>(1);
+ locations.add(location);
+ splitsToAdd.put(r, locations);
+ }
+ }
+ }
+ }
+
+ if (!autoAdjust)
+ for (Entry<Range,ArrayList<String>> entry : splitsToAdd.entrySet())
+ splits.add(new org.apache.accumulo.core.client.mapreduce.RangeInputSplit(entry.getKey(), entry.getValue().toArray(new String[0])));
+
+ for (InputSplit inputSplit : splits) {
+ org.apache.accumulo.core.client.mapreduce.RangeInputSplit split = (org.apache.accumulo.core.client.mapreduce.RangeInputSplit) inputSplit;
+
+ split.setTable(tableName);
+ split.setOffline(offline);
+ split.setIsolatedScan(isolated);
+ split.setUsesLocalIterators(localIterators);
+ split.setMockInstance(mockInstance);
+ split.setFetchedColumns(fetchedColumns);
+ split.setPrincipal(principal);
+ split.setToken(token);
+ split.setInstanceName(instance.getInstanceName());
+ split.setZooKeepers(instance.getZooKeepers());
+ split.setAuths(auths);
+ split.setIterators(iterators);
+ split.setLogLevel(logLevel);
+ }
+
+ return splits;
+ }
+
+ // ----------------------------------------------------------------------------------------------------
+ // Everything below this line is deprecated and should go away in future versions
+ // ----------------------------------------------------------------------------------------------------
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setScanIsolation(Job, boolean)} instead.
+ */
+ @Deprecated
+ public static void setIsolated(Configuration conf, boolean enable) {
+ InputConfigurator.setScanIsolation(CLASS, conf, enable);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setLocalIterators(Job, boolean)} instead.
+ */
+ @Deprecated
+ public static void setLocalIterators(Configuration conf, boolean enable) {
+ InputConfigurator.setLocalIterators(CLASS, conf, enable);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setConnectorInfo(Job, String, AuthenticationToken)}, {@link #setInputTableName(Job, String)}, and
+ * {@link #setScanAuthorizations(Job, Authorizations)} instead.
+ */
+ @Deprecated
+ public static void setInputInfo(Configuration conf, String user, byte[] passwd, String table, Authorizations auths) {
+ try {
+ InputConfigurator.setConnectorInfo(CLASS, conf, user, new PasswordToken(passwd));
+ } catch (AccumuloSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ InputConfigurator.setInputTableName(CLASS, conf, table);
+ InputConfigurator.setScanAuthorizations(CLASS, conf, auths);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setZooKeeperInstance(Job, String, String)} instead.
+ */
+ @Deprecated
+ public static void setZooKeeperInstance(Configuration conf, String instanceName, String zooKeepers) {
+ InputConfigurator.setZooKeeperInstance(CLASS, conf, instanceName, zooKeepers);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setMockInstance(Job, String)} instead.
+ */
+ @Deprecated
+ public static void setMockInstance(Configuration conf, String instanceName) {
+ InputConfigurator.setMockInstance(CLASS, conf, instanceName);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setRanges(Job, Collection)} instead.
+ */
+ @Deprecated
+ public static void setRanges(Configuration conf, Collection<Range> ranges) {
+ InputConfigurator.setRanges(CLASS, conf, ranges);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setAutoAdjustRanges(Job, boolean)} instead.
+ */
+ @Deprecated
+ public static void disableAutoAdjustRanges(Configuration conf) {
+ InputConfigurator.setAutoAdjustRanges(CLASS, conf, false);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #addIterator(Job, IteratorSetting)} to add the {@link VersioningIterator} instead.
+ */
+ @Deprecated
+ public static void setMaxVersions(Configuration conf, int maxVersions) throws IOException {
+ IteratorSetting vers = new IteratorSetting(1, "vers", VersioningIterator.class);
+ try {
+ VersioningIterator.setMaxVersions(vers, maxVersions);
+ } catch (IllegalArgumentException e) {
+ throw new IOException(e);
+ }
+ InputConfigurator.addIterator(CLASS, conf, vers);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setOfflineTableScan(Job, boolean)} instead.
+ */
+ @Deprecated
+ public static void setScanOffline(Configuration conf, boolean scanOff) {
+ InputConfigurator.setOfflineTableScan(CLASS, conf, scanOff);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #fetchColumns(Job, Collection)} instead.
+ */
+ @Deprecated
+ public static void fetchColumns(Configuration conf, Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
+ InputConfigurator.fetchColumns(CLASS, conf, columnFamilyColumnQualifierPairs);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setLogLevel(Job, Level)} instead.
+ */
+ @Deprecated
+ public static void setLogLevel(Configuration conf, Level level) {
+ InputConfigurator.setLogLevel(CLASS, conf, level);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #addIterator(Job, IteratorSetting)} instead.
+ */
+ @Deprecated
+ public static void addIterator(Configuration conf, IteratorSetting cfg) {
+ InputConfigurator.addIterator(CLASS, conf, cfg);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #isIsolated(JobContext)} instead.
+ */
+ @Deprecated
+ protected static boolean isIsolated(Configuration conf) {
+ return InputConfigurator.isIsolated(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #usesLocalIterators(JobContext)} instead.
+ */
+ @Deprecated
+ protected static boolean usesLocalIterators(Configuration conf) {
+ return InputConfigurator.usesLocalIterators(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getPrincipal(JobContext)} instead.
+ */
+ @Deprecated
+ protected static String getPrincipal(Configuration conf) {
+ return InputConfigurator.getPrincipal(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getToken(JobContext)} instead.
+ */
+ @Deprecated
+ protected static byte[] getToken(Configuration conf) {
+ return InputConfigurator.getToken(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getInputTableName(JobContext)} instead.
+ */
+ @Deprecated
+ protected static String getTablename(Configuration conf) {
+ return InputConfigurator.getInputTableName(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getScanAuthorizations(JobContext)} instead.
+ */
+ @Deprecated
+ protected static Authorizations getAuthorizations(Configuration conf) {
+ return InputConfigurator.getScanAuthorizations(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getInstance(JobContext)} instead.
+ */
+ @Deprecated
+ protected static Instance getInstance(Configuration conf) {
+ return InputConfigurator.getInstance(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getTabletLocator(JobContext)} instead.
+ */
+ @Deprecated
+ protected static TabletLocator getTabletLocator(Configuration conf) throws TableNotFoundException {
+ return InputConfigurator.getTabletLocator(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getRanges(JobContext)} instead.
+ */
+ @Deprecated
+ protected static List<Range> getRanges(Configuration conf) throws IOException {
+ return InputConfigurator.getRanges(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getFetchedColumns(JobContext)} instead.
+ */
+ @Deprecated
+ protected static Set<Pair<Text,Text>> getFetchedColumns(Configuration conf) {
+ return InputConfigurator.getFetchedColumns(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getAutoAdjustRanges(JobContext)} instead.
+ */
+ @Deprecated
+ protected static boolean getAutoAdjustRanges(Configuration conf) {
+ return InputConfigurator.getAutoAdjustRanges(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getLogLevel(JobContext)} instead.
+ */
+ @Deprecated
+ protected static Level getLogLevel(Configuration conf) {
+ return InputConfigurator.getLogLevel(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #validateOptions(JobContext)} instead.
+ */
+ @Deprecated
+ protected static void validateOptions(Configuration conf) throws IOException {
+ InputConfigurator.validateOptions(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #addIterator(Job, IteratorSetting)} to add the {@link VersioningIterator} instead.
+ */
+ @Deprecated
+ protected static int getMaxVersions(Configuration conf) {
+ // This is so convoluted, because the only reason to get the number of maxVersions is to construct the same type of IteratorSetting object we have to
+ // deconstruct to get at this option in the first place, but to preserve correct behavior, this appears necessary.
+ List<IteratorSetting> iteratorSettings = InputConfigurator.getIterators(CLASS, conf);
+ for (IteratorSetting setting : iteratorSettings) {
+ if ("vers".equals(setting.getName()) && 1 == setting.getPriority() && VersioningIterator.class.getName().equals(setting.getIteratorClass())) {
+ if (setting.getOptions().containsKey("maxVersions"))
+ return Integer.parseInt(setting.getOptions().get("maxVersions"));
+ else
+ return -1;
+ }
+ }
+ return -1;
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #isOfflineScan(JobContext)} instead.
+ */
+ @Deprecated
+ protected static boolean isOfflineScan(Configuration conf) {
+ return InputConfigurator.isOfflineScan(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getIterators(JobContext)} instead.
+ */
+ @Deprecated
+ protected static List<AccumuloIterator> getIterators(Configuration conf) {
+ List<IteratorSetting> iteratorSettings = InputConfigurator.getIterators(CLASS, conf);
+ List<AccumuloIterator> deprecatedIterators = new ArrayList<AccumuloIterator>(iteratorSettings.size());
+ for (IteratorSetting setting : iteratorSettings) {
+ AccumuloIterator deprecatedIter = new AccumuloIterator(setting.getPriority() + AccumuloIterator.FIELD_SEP + setting.getIteratorClass()
+ + AccumuloIterator.FIELD_SEP + setting.getName());
+ deprecatedIterators.add(deprecatedIter);
+ }
+ return deprecatedIterators;
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getIterators(JobContext)} instead.
+ */
+ @Deprecated
+ protected static List<AccumuloIteratorOption> getIteratorOptions(Configuration conf) {
+ List<IteratorSetting> iteratorSettings = InputConfigurator.getIterators(CLASS, conf);
+ List<AccumuloIteratorOption> deprecatedIteratorOptions = new ArrayList<AccumuloIteratorOption>(iteratorSettings.size());
+ for (IteratorSetting setting : iteratorSettings) {
+ for (Entry<String,String> opt : setting.getOptions().entrySet()) {
+ String deprecatedOption;
+ try {
+ deprecatedOption = setting.getName() + AccumuloIteratorOption.FIELD_SEP + URLEncoder.encode(opt.getKey(), "UTF-8") + AccumuloIteratorOption.FIELD_SEP
+ + URLEncoder.encode(opt.getValue(), "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ deprecatedIteratorOptions.add(new AccumuloIteratorOption(deprecatedOption));
+ }
+ }
+ return deprecatedIteratorOptions;
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link IteratorSetting} instead.
+ */
+ @Deprecated
+ static class AccumuloIterator {
+
+ private static final String FIELD_SEP = ":";
+
+ private int priority;
+ private String iteratorClass;
+ private String iteratorName;
+
+ public AccumuloIterator(int priority, String iteratorClass, String iteratorName) {
+ this.priority = priority;
+ this.iteratorClass = iteratorClass;
+ this.iteratorName = iteratorName;
+ }
+
+ // Parses out a setting given an string supplied from an earlier toString() call
+ public AccumuloIterator(String iteratorSetting) {
+ // Parse the string to expand the iterator
+ StringTokenizer tokenizer = new StringTokenizer(iteratorSetting, FIELD_SEP);
+ priority = Integer.parseInt(tokenizer.nextToken());
+ iteratorClass = tokenizer.nextToken();
+ iteratorName = tokenizer.nextToken();
+ }
+
+ public int getPriority() {
+ return priority;
+ }
+
+ public String getIteratorClass() {
+ return iteratorClass;
+ }
+
+ public String getIteratorName() {
+ return iteratorName;
+ }
+
+ @Override
+ public String toString() {
+ return priority + FIELD_SEP + iteratorClass + FIELD_SEP + iteratorName;
+ }
+
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link IteratorSetting} instead.
+ */
+ @Deprecated
+ static class AccumuloIteratorOption {
+ private static final String FIELD_SEP = ":";
+
+ private String iteratorName;
+ private String key;
+ private String value;
+
+ public AccumuloIteratorOption(String iteratorName, String key, String value) {
+ this.iteratorName = iteratorName;
+ this.key = key;
+ this.value = value;
+ }
+
+ // Parses out an option given a string supplied from an earlier toString() call
+ public AccumuloIteratorOption(String iteratorOption) {
+ StringTokenizer tokenizer = new StringTokenizer(iteratorOption, FIELD_SEP);
+ this.iteratorName = tokenizer.nextToken();
+ try {
+ this.key = URLDecoder.decode(tokenizer.nextToken(), "UTF-8");
+ this.value = URLDecoder.decode(tokenizer.nextToken(), "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public String getIteratorName() {
+ return iteratorName;
+ }
+
+ public String getKey() {
+ return key;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return iteratorName + FIELD_SEP + URLEncoder.encode(key, "UTF-8") + FIELD_SEP + URLEncoder.encode(value, "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+ // use reflection to pull the Configuration out of the JobContext for Hadoop 1 and Hadoop 2 compatibility
+ static Configuration getConfiguration(JobContext context) {
+ try {
+ Class<?> c = InputFormatBase.class.getClassLoader().loadClass("org.apache.hadoop.mapreduce.JobContext");
+ Method m = c.getMethod("getConfiguration");
+ Object o = m.invoke(context, new Object[0]);
+ return (Configuration) o;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * @deprecated since 1.5.2; Use {@link org.apache.accumulo.core.client.mapreduce.RangeInputSplit} instead.
+ * @see org.apache.accumulo.core.client.mapreduce.RangeInputSplit
+ */
+ @Deprecated
+ public static class RangeInputSplit extends org.apache.accumulo.core.client.mapreduce.RangeInputSplit {
+
+ public RangeInputSplit() {
+ super();
+ }
+
+ public RangeInputSplit(Range range, String[] locations) {
+ super(range, locations);
+ }
+ }
+}
[64/64] [abbrv] git commit: Merge branch '1.6.0-SNAPSHOT'
Posted by ct...@apache.org.
Merge branch '1.6.0-SNAPSHOT'
Conflicts:
core/src/main/java/org/apache/accumulo/core/data/Condition.java
core/src/main/java/org/apache/accumulo/core/data/Key.java
core/src/main/java/org/apache/accumulo/core/data/Range.java
test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/46b0f986
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/46b0f986
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/46b0f986
Branch: refs/heads/master
Commit: 46b0f986d7fee32c9b98ea72a97e239944ab7202
Parents: 8856c1f 716ea0e
Author: Christopher Tubbs <ct...@apache.org>
Authored: Wed Apr 9 13:56:59 2014 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Wed Apr 9 13:56:59 2014 -0400
----------------------------------------------------------------------
.../core/client/ClientSideIteratorScanner.java | 2 -
.../accumulo/core/client/ConditionalWriter.java | 2 -
.../core/client/ConditionalWriterConfig.java | 2 -
.../apache/accumulo/core/client/Connector.java | 2 -
.../accumulo/core/client/IteratorSetting.java | 4 -
.../accumulo/core/client/RowIterator.java | 4 -
.../accumulo/core/client/ScannerBase.java | 1 -
.../core/client/admin/ActiveCompaction.java | 1 -
.../core/client/admin/InstanceOperations.java | 12 --
.../core/client/admin/TableOperations.java | 30 ----
.../core/client/admin/TableOperationsImpl.java | 4 -
.../core/client/impl/ThriftTransportPool.java | 16 +-
.../core/client/mapred/AbstractInputFormat.java | 2 -
.../client/mapred/AccumuloOutputFormat.java | 2 -
.../client/mapreduce/AbstractInputFormat.java | 3 +-
.../client/mapreduce/AccumuloOutputFormat.java | 2 -
.../core/client/mapreduce/InputTableConfig.java | 3 -
.../mapreduce/lib/util/ConfiguratorBase.java | 2 -
.../core/client/mock/MockBatchDeleter.java | 4 -
.../java/org/apache/accumulo/core/data/Key.java | 4 +
.../org/apache/accumulo/core/data/Range.java | 6 +-
.../accumulo/core/file/rfile/BlockIndex.java | 5 -
.../accumulo/core/file/rfile/bcfile/BCFile.java | 15 +-
.../core/file/rfile/bcfile/ByteArray.java | 2 -
.../accumulo/core/file/rfile/bcfile/Utils.java | 11 --
.../core/iterators/TypedValueCombiner.java | 6 -
.../core/iterators/ValueFormatException.java | 6 -
.../core/iterators/system/MapFileIterator.java | 8 +-
.../core/iterators/user/GrepIterator.java | 3 -
.../iterators/user/IntersectingIterator.java | 10 --
.../accumulo/core/iterators/user/RowFilter.java | 1 -
.../iterators/user/TransformingIterator.java | 164 +++++++++----------
.../core/iterators/user/VersioningIterator.java | 3 -
.../accumulo/core/security/SecurityUtil.java | 1 -
.../security/crypto/CryptoModuleFactory.java | 1 -
.../security/crypto/CryptoModuleParameters.java | 6 -
.../core/client/impl/ScannerOptionsTest.java | 2 -
.../client/lexicoder/ReverseLexicoderTest.java | 2 -
.../client/mapred/AccumuloInputFormatTest.java | 4 -
.../mapreduce/AccumuloInputFormatTest.java | 4 -
.../core/client/mock/MockNamespacesTest.java | 8 -
.../core/security/VisibilityConstraintTest.java | 3 -
.../simple/client/RandomBatchScanner.java | 5 -
.../simple/client/RandomBatchWriter.java | 4 -
.../simple/client/SequentialBatchWriter.java | 5 -
.../simple/client/TraceDumpExample.java | 9 +-
.../examples/simple/dirlist/QueryUtil.java | 3 -
.../examples/simple/mapreduce/NGramIngest.java | 3 -
.../examples/simple/mapreduce/TableToFile.java | 1 -
.../accumulo/examples/simple/shard/Query.java | 3 -
.../minicluster/MiniAccumuloRunner.java | 2 -
.../accumulo/proxy/TestProxyReadWrite.java | 10 --
.../accumulo/server/conf/ConfigSanityCheck.java | 3 -
.../server/master/balancer/TabletBalancer.java | 2 -
.../server/master/state/TabletStateStore.java | 7 -
.../server/metrics/AbstractMetricsImpl.java | 4 -
.../server/security/handler/ZKAuthorizor.java | 4 -
.../server/security/handler/ZKPermHandler.java | 4 -
.../accumulo/server/util/LoginProperties.java | 3 -
.../accumulo/server/util/RestoreZookeeper.java | 4 -
.../accumulo/server/util/TableDiskUsage.java | 3 -
.../accumulo/server/util/TabletServerLocks.java | 3 -
.../org/apache/accumulo/tserver/MemValue.java | 4 +-
.../apache/accumulo/tserver/TabletServer.java | 14 +-
.../tserver/compaction/CompactionStrategy.java | 4 -
.../accumulo/tserver/logger/LogReader.java | 1 -
.../start/classloader/AccumuloClassLoader.java | 3 -
.../start/classloader/vfs/ContextManager.java | 2 -
.../vfs/PostDelegatingVFSClassLoader.java | 7 +-
.../vfs/providers/HdfsFileSystem.java | 5 -
.../accumulo/test/NativeMapPerformanceTest.java | 3 -
.../accumulo/test/NativeMapStressTest.java | 3 -
.../test/continuous/ContinuousMoru.java | 1 -
.../test/continuous/ContinuousVerify.java | 1 -
.../test/functional/CacheTestClean.java | 3 -
.../accumulo/test/randomwalk/Environment.java | 6 +-
.../accumulo/test/randomwalk/Framework.java | 3 -
.../apache/accumulo/test/randomwalk/Node.java | 1 -
.../randomwalk/concurrent/CheckBalance.java | 5 +-
79 files changed, 125 insertions(+), 396 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/core/src/main/java/org/apache/accumulo/core/client/ConditionalWriterConfig.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/ConditionalWriterConfig.java
index 064b9be,a220e62..7671c35
--- a/core/src/main/java/org/apache/accumulo/core/client/ConditionalWriterConfig.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ConditionalWriterConfig.java
@@@ -42,11 -42,9 +42,9 @@@ public class ConditionalWriterConfig
*
* <p>
* Any condition that is not visible with this set of authorizations will fail.
- *
- * @param auths
*/
public ConditionalWriterConfig setAuthorizations(Authorizations auths) {
- ArgumentChecker.notNull(auths);
+ checkArgument(auths != null, "auths is null");
this.auths = auths;
return this;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
index ca50ce2,e69f3dd..b966a4a
--- a/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
@@@ -85,11 -85,9 +85,9 @@@ public class IteratorSetting implement
/**
* Set the iterator's name. Must be a simple alphanumeric identifier.
- *
- * @param name
*/
public void setName(String name) {
- ArgumentChecker.notNull(name);
+ checkArgument(name != null, "name is null");
this.name = name;
}
@@@ -105,11 -103,9 +103,9 @@@
/**
* Set the name of the class that implements the iterator. The class does not have to be present on the client, but it must be available to all tablet
* servers.
- *
- * @param iteratorClass
*/
public void setIteratorClass(String iteratorClass) {
- ArgumentChecker.notNull(iteratorClass);
+ checkArgument(iteratorClass != null, "iteratorClass is null");
this.iteratorClass = iteratorClass;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/core/src/main/java/org/apache/accumulo/core/data/Key.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Key.java
index 41754b5,2b44359..bfeb095
--- a/core/src/main/java/org/apache/accumulo/core/data/Key.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Key.java
@@@ -715,12 -638,13 +717,13 @@@ public class Key implements WritableCom
}
/**
- * Compare all elements of a key. The elements (row, column family, column qualifier, column visibility, timestamp, and delete marker) are compared in order
- * until an unequal element is found. If the row is equal, then compare the column family, etc. The row, column family, column qualifier, and column
- * visibility are compared lexographically and sorted ascending. The timestamps are compared numerically and sorted descending so that the most recent data
- * comes first. Lastly, a delete marker of true sorts before a delete marker of false.
+ * Compares this key with another.
+ *
+ * @param other key to compare to
+ * @return comparison result
+ * @see #compareTo(Key, PartialKey)
*/
-
+ @Override
public int compareTo(Key other) {
return compareTo(other, PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME_DEL);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/core/src/main/java/org/apache/accumulo/core/data/Range.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Range.java
index a9469a8,412ee53..56e823e
--- a/core/src/main/java/org/apache/accumulo/core/data/Range.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Range.java
@@@ -314,12 -310,10 +314,13 @@@ public class Range implements WritableC
}
/**
- * Compares this range to another range. Compares in the order start key, inclusiveness of start key, end key, inclusiveness of end key. Infinite keys sort
+ * Compares this range to another range. Compares in order: start key, inclusiveness of start key, end key, inclusiveness of end key. Infinite keys sort
* first, and non-infinite keys are compared with {@link Key#compareTo(Key)}. Inclusive sorts before non-inclusive.
+ *
+ * @param o range to compare
+ * @return comparison result
*/
+ @Override
public int compareTo(Range o) {
int comp;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/test/src/main/java/org/apache/accumulo/test/randomwalk/Environment.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/Environment.java
index b99ee77,0000000..72792ef
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Environment.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Environment.java
@@@ -1,215 -1,0 +1,213 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.randomwalk;
+
+import static com.google.common.base.Preconditions.checkNotNull;
++
+import java.lang.management.ManagementFactory;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
++
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.log4j.Logger;
+
+/**
+ * The test environment that is available for randomwalk tests. This includes configuration properties that are available to any randomwalk test and facilities
+ * for creating client-side objects. This class is not thread-safe.
+ */
+public class Environment {
+ /**
+ * The configuration property key for a username.
+ */
+ public static final String KEY_USERNAME = "USERNAME";
+ /**
+ * The configuration property key for a password.
+ */
+ public static final String KEY_PASSWORD = "PASSWORD";
+ /**
+ * The configuration property key for the instance name.
+ */
+ public static final String KEY_INSTANCE = "INSTANCE";
+ /**
+ * The configuration property key for the comma-separated list of ZooKeepers.
+ */
+ public static final String KEY_ZOOKEEPERS = "ZOOKEEPERS";
+ /**
+ * The configuration property key for the maximum memory for the multi-table batch writer.
+ */
+ public static final String KEY_MAX_MEM = "MAX_MEM";
+ /**
+ * The configuration property key for the maximum latency, in milliseconds, for the multi-table batch writer.
+ */
+ public static final String KEY_MAX_LATENCY = "MAX_LATENCY";
+ /**
+ * The configuration property key for the number of write threads for the multi-table batch writer.
+ */
+ public static final String KEY_NUM_THREADS = "NUM_THREADS";
+
+ private static final Logger log = Logger.getLogger(Environment.class);
+
+ private final Properties p;
+ private Instance instance = null;
+ private Connector connector = null;
+ private MultiTableBatchWriter mtbw = null;
+
+ /**
+ * Creates a new test environment.
+ *
+ * @param p
+ * configuration properties
+ * @throws NullPointerException
+ * if p is null
+ */
+ Environment(Properties p) {
+ checkNotNull(p);
+ this.p = p;
+ }
+
+ /**
+ * Gets a copy of the configuration properties.
+ *
+ * @return a copy of the configuration properties
+ */
+ Properties copyConfigProperties() {
+ return new Properties(p);
+ }
+
+ /**
+ * Gets a configuration property.
+ *
+ * @param key
+ * key
+ * @return property value
+ */
+ public String getConfigProperty(String key) {
+ return p.getProperty(key);
+ }
+
+ /**
+ * Gets the configured username.
+ *
+ * @return username
+ */
+ public String getUserName() {
+ return p.getProperty(KEY_USERNAME);
+ }
+
+ /**
+ * Gets the configured password.
+ *
+ * @return password
+ */
+ public String getPassword() {
+ return p.getProperty(KEY_PASSWORD);
+ }
+
+ /**
+ * Gets this process's ID.
+ *
+ * @return pid
+ */
+ public String getPid() {
+ return ManagementFactory.getRuntimeMXBean().getName().split("@")[0];
+ }
+
+ /**
+ * Gets an authentication token based on the configured password.
+ *
+ * @return authentication token
+ */
+ public AuthenticationToken getToken() {
+ return new PasswordToken(getPassword());
+ }
+
+ /**
+ * Gets an Accumulo instance object. The same instance is reused after the first call.
+ *
+ * @return instance
+ */
+ public Instance getInstance() {
+ if (instance == null) {
+ String instance = p.getProperty(KEY_INSTANCE);
+ String zookeepers = p.getProperty(KEY_ZOOKEEPERS);
+ this.instance = new ZooKeeperInstance(new ClientConfiguration().withInstance(instance).withZkHosts(zookeepers));
+ }
+ return instance;
+ }
+
+ /**
+ * Gets an Accumulo connector. The same connector is reused after the first call.
+ *
+ * @return connector
- * @throws AccumuloException
- * @throws AccumuloSecurityException
+ */
+ public Connector getConnector() throws AccumuloException, AccumuloSecurityException {
+ if (connector == null) {
+ connector = getInstance().getConnector(getUserName(), getToken());
+ }
+ return connector;
+ }
+
+ /**
+ * Gets a multitable batch writer. The same object is reused after the first call unless it is reset.
+ *
+ * @return multitable batch writer
+ * @throws NumberFormatException
+ * if any of the numeric batch writer configuration properties cannot be parsed
- * @throws AccumuloException
- * @throws AccumuloSecurityException
+ * @throws NumberFormatException
+ * if any configuration property cannot be parsed
+ */
+ public MultiTableBatchWriter getMultiTableBatchWriter() throws AccumuloException, AccumuloSecurityException {
+ if (mtbw == null) {
+ long maxMem = Long.parseLong(p.getProperty(KEY_MAX_MEM));
+ long maxLatency = Long.parseLong(p.getProperty(KEY_MAX_LATENCY));
+ int numThreads = Integer.parseInt(p.getProperty(KEY_NUM_THREADS));
+ mtbw = getConnector().createMultiTableBatchWriter(
+ new BatchWriterConfig().setMaxMemory(maxMem).setMaxLatency(maxLatency, TimeUnit.MILLISECONDS).setMaxWriteThreads(numThreads));
+ }
+ return mtbw;
+ }
+
+ /**
+ * Checks if a multitable batch writer has been created by this wrapper.
+ *
+ * @return true if multitable batch writer is already created
+ */
+ public boolean isMultiTableBatchWriterInitialized() {
+ return mtbw != null;
+ }
+
+ /**
+ * Clears the multitable batch writer previously created and remembered by this wrapper.
+ */
+ public void resetMultiTableBatchWriter() {
+ if (mtbw == null)
+ return;
+ if (!mtbw.isClosed()) {
+ log.warn("Setting non-closed MultiTableBatchWriter to null (leaking resources)");
+ }
+ mtbw = null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/test/src/main/java/org/apache/accumulo/test/randomwalk/Framework.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/Framework.java
index 56fb366,7cb58c9..0e54d90
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Framework.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Framework.java
@@@ -49,10 -53,8 +49,8 @@@ public class Framework
*
* @param startName
* Full name of starting graph or test
- * @param state
- * @param confDir
*/
- public int run(String startName, State state, String confDir) {
+ public int run(String startName, State state, Environment env, String confDir) {
try {
System.out.println("confDir " + confDir);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
index 980c608,b74b6cd..1588d5a
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
@@@ -34,11 -33,8 +34,10 @@@ public abstract class Node
*
* @param state
* Random walk state passed between nodes
+ * @param env
+ * test environment
- * @throws Exception
*/
- public abstract void visit(State state, Properties props) throws Exception;
+ public abstract void visit(State state, Environment env, Properties props) throws Exception;
@Override
public boolean equals(Object o) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46b0f986/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
index 0335b49,8dc2b6e..df246d4
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
@@@ -40,14 -39,11 +40,11 @@@ public class CheckBalance extends Test
static final String LAST_UNBALANCED_TIME = "lastUnbalancedTime";
static final String UNBALANCED_COUNT = "unbalancedCount";
- /* (non-Javadoc)
- * @see org.apache.accumulo.test.randomwalk.Node#visit(org.apache.accumulo.test.randomwalk.State, java.util.Properties)
- */
@Override
- public void visit(State state, Properties props) throws Exception {
+ public void visit(State state, Environment env, Properties props) throws Exception {
log.debug("checking balance");
Map<String,Long> counts = new HashMap<String,Long>();
- Scanner scanner = state.getConnector().createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ Scanner scanner = env.getConnector().createScanner(MetadataTable.NAME, Authorizations.EMPTY);
scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
for (Entry<Key,Value> entry : scanner) {
String location = entry.getKey().getColumnQualifier().toString();
[34/64] [abbrv] git commit: Merge branch '1.4.6-SNAPSHOT' into
1.5.2-SNAPSHOT
Posted by ct...@apache.org.
Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Conflicts:
core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java
core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java
core/src/main/java/org/apache/accumulo/core/iterators/FirstEntryInRowIterator.java
core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
server/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
server/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
server/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
src/core/src/main/java/org/apache/accumulo/core/file/map/MapFileUtil.java
src/server/src/main/java/org/apache/accumulo/server/util/DumpMapFile.java
src/server/src/main/java/org/apache/accumulo/server/util/DumpTabletsOnServer.java
test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/92613388
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/92613388
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/92613388
Branch: refs/heads/1.5.2-SNAPSHOT
Commit: 92613388919b6fc138e829382cc1cbb6647faa31
Parents: 5363d78 c8e165a
Author: Christopher Tubbs <ct...@apache.org>
Authored: Wed Apr 9 13:14:57 2014 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Wed Apr 9 13:14:57 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/Constants.java | 1 -
.../core/client/ClientSideIteratorScanner.java | 2 -
.../apache/accumulo/core/client/Connector.java | 2 -
.../accumulo/core/client/IteratorSetting.java | 4 -
.../accumulo/core/client/RowIterator.java | 4 -
.../accumulo/core/client/ScannerBase.java | 2 +-
.../accumulo/core/client/ZooKeeperInstance.java | 2 -
.../core/client/admin/ActiveCompaction.java | 1 -
.../core/client/admin/InstanceOperations.java | 12 -
.../client/admin/InstanceOperationsImpl.java | 33 --
.../core/client/admin/TableOperations.java | 32 --
.../core/client/admin/TableOperationsImpl.java | 4 -
.../core/client/impl/OfflineScanner.java | 6 -
.../core/client/impl/ThriftTransportPool.java | 16 +-
.../client/mapred/AccumuloOutputFormat.java | 1 -
.../core/client/mapred/InputFormatBase.java | 1 -
.../client/mapreduce/AccumuloOutputFormat.java | 1 -
.../core/client/mapreduce/InputFormatBase.java | 1 -
.../mapreduce/lib/util/ConfiguratorBase.java | 1 -
.../core/client/mock/MockBatchDeleter.java | 4 -
.../client/mock/MockInstanceOperations.java | 43 ---
.../apache/accumulo/core/data/ColumnUpdate.java | 1 -
.../java/org/apache/accumulo/core/data/Key.java | 7 +-
.../apache/accumulo/core/data/KeyExtent.java | 6 +-
.../org/apache/accumulo/core/data/Range.java | 17 +-
.../accumulo/core/file/rfile/BlockIndex.java | 5 -
.../accumulo/core/file/rfile/bcfile/BCFile.java | 14 +-
.../core/file/rfile/bcfile/ByteArray.java | 2 -
.../accumulo/core/file/rfile/bcfile/Chunk.java | 2 -
.../accumulo/core/file/rfile/bcfile/TFile.java | 44 ---
.../core/file/rfile/bcfile/TFileDumper.java | 1 -
.../accumulo/core/file/rfile/bcfile/Utils.java | 11 -
.../core/iterators/TypedValueCombiner.java | 6 -
.../core/iterators/ValueFormatException.java | 6 -
.../core/iterators/system/MapFileIterator.java | 8 +-
.../core/iterators/user/GrepIterator.java | 3 -
.../iterators/user/IntersectingIterator.java | 10 -
.../accumulo/core/iterators/user/RowFilter.java | 1 -
.../iterators/user/TransformingIterator.java | 166 +++++-----
.../core/iterators/user/VersioningIterator.java | 3 -
.../accumulo/core/security/SecurityUtil.java | 1 -
.../core/security/crypto/CryptoModule.java | 2 -
.../security/crypto/CryptoModuleFactory.java | 1 -
.../core/client/impl/ScannerOptionsTest.java | 2 -
.../client/mapred/AccumuloInputFormatTest.java | 4 -
.../mapreduce/AccumuloInputFormatTest.java | 6 -
.../shell/command/FormatterCommandTest.java | 15 +-
.../simple/client/RandomBatchScanner.java | 5 -
.../simple/client/RandomBatchWriter.java | 4 -
.../simple/client/SequentialBatchWriter.java | 5 -
.../simple/client/TraceDumpExample.java | 9 +-
.../examples/simple/dirlist/QueryUtil.java | 3 -
.../examples/simple/mapreduce/NGramIngest.java | 3 -
.../examples/simple/mapreduce/TableToFile.java | 1 -
.../accumulo/examples/simple/shard/Query.java | 3 -
.../minicluster/MiniAccumuloCluster.java | 10 -
.../accumulo/proxy/TestProxyReadWrite.java | 10 -
.../accumulo/server/conf/ConfigSanityCheck.java | 3 -
.../accumulo/server/logger/LogReader.java | 1 -
.../master/balancer/ChaoticLoadBalancer.java | 8 -
.../server/master/balancer/TabletBalancer.java | 2 -
.../server/master/state/TabletStateStore.java | 8 +-
.../server/master/tableOps/TraceRepo.java | 25 --
.../server/metanalysis/LogFileOutputFormat.java | 4 -
.../server/metanalysis/PrintEvents.java | 5 +-
.../server/metrics/AbstractMetricsImpl.java | 4 -
.../security/handler/InsecurePermHandler.java | 42 ---
.../server/security/handler/ZKAuthorizor.java | 7 +-
.../server/security/handler/ZKPermHandler.java | 6 +-
.../accumulo/server/tabletserver/MemValue.java | 4 +-
.../server/tabletserver/TabletServer.java | 8 +-
.../server/util/AddFilesWithMissingEntries.java | 1 -
.../accumulo/server/util/DumpZookeeper.java | 3 -
.../server/util/FindOfflineTablets.java | 3 -
.../accumulo/server/util/LoginProperties.java | 3 -
.../accumulo/server/util/MetadataTable.java | 305 +++++++++----------
.../accumulo/server/util/RestoreZookeeper.java | 4 -
.../accumulo/server/util/SendLogToChainsaw.java | 1 -
.../accumulo/server/util/TServerUtils.java | 7 +-
.../accumulo/server/util/TableDiskUsage.java | 3 -
.../accumulo/server/util/TabletServerLocks.java | 3 -
.../start/classloader/AccumuloClassLoader.java | 3 -
.../start/classloader/vfs/ContextManager.java | 2 -
.../vfs/PostDelegatingVFSClassLoader.java | 7 +-
.../vfs/providers/HdfsFileSystem.java | 5 -
.../apache/accumulo/test/GetMasterStats.java | 9 -
.../accumulo/test/NativeMapPerformanceTest.java | 3 -
.../accumulo/test/NativeMapStressTest.java | 3 -
.../test/continuous/ContinuousMoru.java | 1 -
.../test/continuous/ContinuousVerify.java | 1 -
.../test/functional/CacheTestClean.java | 3 -
.../accumulo/test/functional/RunTests.java | 4 -
.../metadata/MetadataBatchScanTest.java | 3 -
.../test/performance/thrift/NullTserver.java | 8 +-
.../accumulo/test/randomwalk/Framework.java | 3 -
.../apache/accumulo/test/randomwalk/Node.java | 1 -
.../randomwalk/concurrent/CheckBalance.java | 5 +-
.../instrument/receivers/ZooSpanClient.java | 10 -
98 files changed, 284 insertions(+), 817 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/Constants.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/Constants.java
index 095319e,0000000..66c4034
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/Constants.java
+++ b/core/src/main/java/org/apache/accumulo/core/Constants.java
@@@ -1,213 -1,0 +1,212 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core;
+
+import java.nio.charset.Charset;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+
+public class Constants {
+ public static final Charset UTF8 = Charset.forName("UTF-8");
+ public static final String VERSION = FilteredConstants.VERSION;
+
+ // versions should never be negative
+ public static final Integer WIRE_VERSION = 2;
+ public static final int DATA_VERSION = 5;
+ public static final int PREV_DATA_VERSION = 4;
+
+ // Zookeeper locations
+ public static final String ZROOT = "/accumulo";
+ public static final String ZINSTANCES = "/instances";
+
+ public static final String ZTABLES = "/tables";
+ public static final byte[] ZTABLES_INITIAL_ID = new byte[] {'0'};
+ public static final String ZTABLE_NAME = "/name";
+ public static final String ZTABLE_CONF = "/conf";
+ public static final String ZTABLE_STATE = "/state";
+ public static final String ZTABLE_FLUSH_ID = "/flush-id";
+ public static final String ZTABLE_COMPACT_ID = "/compact-id";
+ public static final String ZTABLE_COMPACT_CANCEL_ID = "/compact-cancel-id";
+
+ public static final String ZROOT_TABLET = "/root_tablet";
+ public static final String ZROOT_TABLET_LOCATION = ZROOT_TABLET + "/location";
+ public static final String ZROOT_TABLET_FUTURE_LOCATION = ZROOT_TABLET + "/future_location";
+ public static final String ZROOT_TABLET_LAST_LOCATION = ZROOT_TABLET + "/lastlocation";
+ public static final String ZROOT_TABLET_WALOGS = ZROOT_TABLET + "/walogs";
+
+ public static final String ZMASTERS = "/masters";
+ public static final String ZMASTER_LOCK = ZMASTERS + "/lock";
+ public static final String ZMASTER_GOAL_STATE = ZMASTERS + "/goal_state";
+ public static final String ZGC = "/gc";
+ public static final String ZGC_LOCK = ZGC + "/lock";
+
+ public static final String ZMONITOR = "/monitor";
+ public static final String ZMONITOR_LOCK = ZMONITOR + "/lock";
+ public static final String ZMONITOR_HTTP_ADDR = ZMONITOR + "/http_addr";
+ public static final String ZMONITOR_LOG4J_ADDR = ZMONITOR + "/log4j_addr";
+
+ public static final String ZCONFIG = "/config";
+
+ public static final String ZTSERVERS = "/tservers";
+
+ public static final String ZDEAD = "/dead";
+ public static final String ZDEADTSERVERS = "/dead/tservers";
+
+ public static final String ZTRACERS = "/tracers";
+
+ public static final String ZPROBLEMS = "/problems";
+ public static final String ZUSERS = "/users";
+
+ public static final String BULK_ARBITRATOR_TYPE = "bulkTx";
+
+ public static final String ZFATE = "/fate";
+
+ public static final String ZNEXT_FILE = "/next_file";
+
+ public static final String ZBULK_FAILED_COPYQ = "/bulk_failed_copyq";
+
+ public static final String ZHDFS_RESERVATIONS = "/hdfs_reservations";
+ public static final String ZRECOVERY = "/recovery";
+
+ public static final String METADATA_TABLE_ID = "!0";
+ public static final String METADATA_TABLE_NAME = "!METADATA";
+ public static final String DEFAULT_TABLET_LOCATION = "/default_tablet";
+ public static final String TABLE_TABLET_LOCATION = "/table_info";
+ public static final String ZTABLE_LOCKS = "/table_locks";
+
+ // reserved keyspace is any row that begins with a tilde '~' character
+ public static final Key METADATA_RESERVED_KEYSPACE_START_KEY = new Key(new Text(new byte[] {'~'}));
+ public static final Key METADATA_RESERVED_KEYSPACE_STOP_KEY = new Key(new Text(new byte[] {'~' + 1}));
+ public static final Range METADATA_RESERVED_KEYSPACE = new Range(METADATA_RESERVED_KEYSPACE_START_KEY, true, METADATA_RESERVED_KEYSPACE_STOP_KEY, false);
+ public static final String METADATA_DELETE_FLAG_PREFIX = "~del";
+ public static final String METADATA_DELETE_FLAG_FOR_METADATA_PREFIX = "!!" + METADATA_DELETE_FLAG_PREFIX;
+ public static final Range METADATA_DELETES_KEYSPACE = new Range(new Key(new Text(METADATA_DELETE_FLAG_PREFIX)), true, new Key(new Text("~dem")), false);
+ public static final Range METADATA_DELETES_FOR_METADATA_KEYSPACE = new Range(new Key(new Text(METADATA_DELETE_FLAG_FOR_METADATA_PREFIX)), true, new Key(new Text("!!~dem")), false);
+ public static final String METADATA_BLIP_FLAG_PREFIX = "~blip"; // BLIP = bulk load in progress
+ public static final Range METADATA_BLIP_KEYSPACE = new Range(new Key(new Text(METADATA_BLIP_FLAG_PREFIX)), true, new Key(new Text("~bliq")), false);
+
+ public static final Text METADATA_SERVER_COLUMN_FAMILY = new Text("srv");
+ public static final Text METADATA_TABLET_COLUMN_FAMILY = new Text("~tab"); // this needs to sort after all other column families for that tablet
+ public static final Text METADATA_CURRENT_LOCATION_COLUMN_FAMILY = new Text("loc");
+ public static final Text METADATA_FUTURE_LOCATION_COLUMN_FAMILY = new Text("future");
+ public static final Text METADATA_LAST_LOCATION_COLUMN_FAMILY = new Text("last");
+ public static final Text METADATA_BULKFILE_COLUMN_FAMILY = new Text("loaded"); // temporary marker that indicates a tablet loaded a bulk file
+ public static final Text METADATA_CLONED_COLUMN_FAMILY = new Text("!cloned"); // temporary marker that indicates a tablet was successfully cloned
+
+ // README : very important that prevRow sort last to avoid race conditions between
+ // garbage collector and split
+ public static final ColumnFQ METADATA_PREV_ROW_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("~pr")); // this needs to sort after everything
+ // else for that tablet
+ public static final ColumnFQ METADATA_OLD_PREV_ROW_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("oldprevrow"));
+ public static final ColumnFQ METADATA_DIRECTORY_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("dir"));
+ public static final ColumnFQ METADATA_TIME_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("time"));
+ public static final ColumnFQ METADATA_FLUSH_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("flush"));
+ public static final ColumnFQ METADATA_COMPACT_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("compact"));
+ public static final ColumnFQ METADATA_SPLIT_RATIO_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("splitRatio"));
+ public static final ColumnFQ METADATA_LOCK_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("lock"));
+
+ public static final Text METADATA_DATAFILE_COLUMN_FAMILY = new Text("file");
+ public static final Text METADATA_SCANFILE_COLUMN_FAMILY = new Text("scan");
+ public static final Text METADATA_LOG_COLUMN_FAMILY = new Text("log");
+ public static final Text METADATA_CHOPPED_COLUMN_FAMILY = new Text("chopped");
+ public static final ColumnFQ METADATA_CHOPPED_COLUMN = new ColumnFQ(METADATA_CHOPPED_COLUMN_FAMILY, new Text("chopped"));
+
+ public static final Range NON_ROOT_METADATA_KEYSPACE = new Range(
+ new Key(KeyExtent.getMetadataEntry(new Text(METADATA_TABLE_ID), null)).followingKey(PartialKey.ROW), true, METADATA_RESERVED_KEYSPACE_START_KEY, false);
+ public static final Range METADATA_KEYSPACE = new Range(new Key(new Text(METADATA_TABLE_ID)), true, METADATA_RESERVED_KEYSPACE_START_KEY, false);
+
+ public static final KeyExtent ROOT_TABLET_EXTENT = new KeyExtent(new Text(METADATA_TABLE_ID), KeyExtent.getMetadataEntry(new Text(METADATA_TABLE_ID), null),
+ null);
+ public static final Range METADATA_ROOT_TABLET_KEYSPACE = new Range(ROOT_TABLET_EXTENT.getMetadataEntry(), false, KeyExtent.getMetadataEntry(new Text(
+ METADATA_TABLE_ID), null), true);
+
+ public static final String VALUE_ENCODING = "UTF-8";
+
+ public static final String BULK_PREFIX = "b-";
+ public static final String OLD_BULK_PREFIX = "bulk_";
+
+ // note: all times are in milliseconds
+
+ public static final int SCAN_BATCH_SIZE = 1000; // this affects the table client caching of metadata
+
+ public static final long MIN_MASTER_LOOP_TIME = 1000;
+ public static final int MASTER_TABLETSERVER_CONNECTION_TIMEOUT = 3000;
+ public static final long CLIENT_SLEEP_BEFORE_RECONNECT = 1000;
+
+ // Security configuration
+ public static final String PW_HASH_ALGORITHM = "SHA-256";
+
+ // Representation of an empty set of authorizations
+ // (used throughout the code, because scans of metadata table and many tests do not set record-level visibility)
+ public static final Authorizations NO_AUTHS = new Authorizations();
+
+ public static final int DEFAULT_MINOR_COMPACTION_MAX_SLEEP_TIME = 60 * 3; // in seconds
+
+ public static final int MAX_DATA_TO_PRINT = 64;
+ public static final int CLIENT_RETRIES = 5;
+ public static final int TSERV_MINC_MAXCONCURRENT_NUMWAITING_MULTIPLIER = 2;
+ public static final String CORE_PACKAGE_NAME = "org.apache.accumulo.core";
+ public static final String OLD_PACKAGE_NAME = "cloudbase";
+ public static final String VALID_TABLE_NAME_REGEX = "^\\w+$";
+ public static final String MAPFILE_EXTENSION = "map";
+ public static final String GENERATED_TABLET_DIRECTORY_PREFIX = "t-";
+
+ public static final String EXPORT_METADATA_FILE = "metadata.bin";
+ public static final String EXPORT_TABLE_CONFIG_FILE = "table_config.txt";
+ public static final String EXPORT_FILE = "exportMetadata.zip";
+ public static final String EXPORT_INFO_FILE = "accumulo_export_info.txt";
+
+ public static String getBaseDir(final AccumuloConfiguration conf) {
+ return conf.get(Property.INSTANCE_DFS_DIR);
+ }
+
+ public static String getTablesDir(final AccumuloConfiguration conf) {
+ return getBaseDir(conf) + "/tables";
+ }
+
+ public static String getRecoveryDir(final AccumuloConfiguration conf) {
+ return getBaseDir(conf) + "/recovery";
+ }
+
+ public static Path getDataVersionLocation(final AccumuloConfiguration conf) {
+ return new Path(getBaseDir(conf) + "/version");
+ }
+
+ public static String getMetadataTableDir(final AccumuloConfiguration conf) {
+ return getTablesDir(conf) + "/" + METADATA_TABLE_ID;
+ }
+
+ public static String getRootTabletDir(final AccumuloConfiguration conf) {
+ return getMetadataTableDir(conf) + ZROOT_TABLET;
+ }
+
+ /**
- * @param conf
+ * @return The write-ahead log directory.
+ */
+ public static String getWalDirectory(final AccumuloConfiguration conf) {
+ return getBaseDir(conf) + "/wal";
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
index 3085f56,0000000..fbf8670
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
@@@ -1,257 -1,0 +1,255 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.impl.ScannerOptions;
+import org.apache.accumulo.core.client.mock.IteratorAdapter;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Column;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.hadoop.io.Text;
+
+/**
+ * A scanner that instantiates iterators on the client side instead of on the tablet server. This can be useful for testing iterators or in cases where you
+ * don't want iterators affecting the performance of tablet servers.<br>
+ * <br>
+ * Suggested usage:<br>
+ * <code>Scanner scanner = new ClientSideIteratorScanner(connector.createScanner(tableName, authorizations));</code><br>
+ * <br>
+ * Iterators added to this scanner will be run in the client JVM. Separate scan iterators can be run on the server side and client side by adding iterators to
+ * the source scanner (which will execute server side) and to the client side scanner (which will execute client side).
+ */
+public class ClientSideIteratorScanner extends ScannerOptions implements Scanner {
+ private int size;
+
+ private Range range;
+ private boolean isolated = false;
+
+ /**
+ * A class that wraps a Scanner in a SortedKeyValueIterator so that other accumulo iterators can use it as a source.
+ */
+ public class ScannerTranslator implements SortedKeyValueIterator<Key,Value> {
+ protected Scanner scanner;
+ Iterator<Entry<Key,Value>> iter;
+ Entry<Key,Value> top = null;
+
+ /**
+ * Constructs an accumulo iterator from a scanner.
+ *
+ * @param scanner
+ * the scanner to iterate over
+ */
+ public ScannerTranslator(final Scanner scanner) {
+ this.scanner = scanner;
+ }
+
+ @Override
+ public void init(final SortedKeyValueIterator<Key,Value> source, final Map<String,String> options, final IteratorEnvironment env) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean hasTop() {
+ return top != null;
+ }
+
+ @Override
+ public void next() throws IOException {
+ if (iter.hasNext())
+ top = iter.next();
+ else
+ top = null;
+ }
+
+ @Override
+ public void seek(final Range range, final Collection<ByteSequence> columnFamilies, final boolean inclusive) throws IOException {
+ if (!inclusive && columnFamilies.size() > 0) {
+ throw new IllegalArgumentException();
+ }
+ scanner.setRange(range);
+ scanner.clearColumns();
+ for (ByteSequence colf : columnFamilies) {
+ scanner.fetchColumnFamily(new Text(colf.toArray()));
+ }
+ iter = scanner.iterator();
+ next();
+ }
+
+ @Override
+ public Key getTopKey() {
+ return top.getKey();
+ }
+
+ @Override
+ public Value getTopValue() {
+ return top.getValue();
+ }
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(final IteratorEnvironment env) {
+ return new ScannerTranslator(scanner);
+ }
+ }
+
+ private ScannerTranslator smi;
+
+ /**
+ * Constructs a scanner that can execute client-side iterators.
+ *
+ * @param scanner
+ * the source scanner
+ */
+ public ClientSideIteratorScanner(final Scanner scanner) {
+ smi = new ScannerTranslator(scanner);
+ this.range = scanner.getRange();
+ this.size = scanner.getBatchSize();
+ this.timeOut = scanner.getTimeout(TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Sets the source Scanner.
- *
- * @param scanner
+ */
+ public void setSource(final Scanner scanner) {
+ smi = new ScannerTranslator(scanner);
+ }
+
+ @Override
+ public Iterator<Entry<Key,Value>> iterator() {
+ smi.scanner.setBatchSize(size);
+ smi.scanner.setTimeout(timeOut, TimeUnit.MILLISECONDS);
+ if (isolated)
+ smi.scanner.enableIsolation();
+ else
+ smi.scanner.disableIsolation();
+
+ final TreeMap<Integer,IterInfo> tm = new TreeMap<Integer,IterInfo>();
+
+ for (IterInfo iterInfo : serverSideIteratorList) {
+ tm.put(iterInfo.getPriority(), iterInfo);
+ }
+
+ SortedKeyValueIterator<Key,Value> skvi;
+ try {
+ skvi = IteratorUtil.loadIterators(smi, tm.values(), serverSideIteratorOptions, new IteratorEnvironment() {
+ @Override
+ public SortedKeyValueIterator<Key,Value> reserveMapFileReader(final String mapFileName) throws IOException {
+ return null;
+ }
+
+ @Override
+ public AccumuloConfiguration getConfig() {
+ return null;
+ }
+
+ @Override
+ public IteratorScope getIteratorScope() {
+ return null;
+ }
+
+ @Override
+ public boolean isFullMajorCompaction() {
+ return false;
+ }
+
+ @Override
+ public void registerSideChannel(final SortedKeyValueIterator<Key,Value> iter) {}
+ }, false, null);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ final Set<ByteSequence> colfs = new TreeSet<ByteSequence>();
+ for (Column c : this.getFetchedColumns()) {
+ colfs.add(new ArrayByteSequence(c.getColumnFamily()));
+ }
+
+ try {
+ skvi.seek(range, colfs, true);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ return new IteratorAdapter(skvi);
+ }
+
+ @Deprecated
+ @Override
+ public void setTimeOut(int timeOut) {
+ if (timeOut == Integer.MAX_VALUE)
+ setTimeout(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
+ else
+ setTimeout(timeOut, TimeUnit.SECONDS);
+ }
+
+ @Deprecated
+ @Override
+ public int getTimeOut() {
+ long timeout = getTimeout(TimeUnit.SECONDS);
+ if (timeout >= Integer.MAX_VALUE)
+ return Integer.MAX_VALUE;
+ return (int) timeout;
+ }
+
+ @Override
+ public void setRange(final Range range) {
+ this.range = range;
+ }
+
+ @Override
+ public Range getRange() {
+ return range;
+ }
+
+ @Override
+ public void setBatchSize(final int size) {
+ this.size = size;
+ }
+
+ @Override
+ public int getBatchSize() {
+ return size;
+ }
+
+ @Override
+ public void enableIsolation() {
+ this.isolated = true;
+ }
+
+ @Override
+ public void disableIsolation() {
+ this.isolated = false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/Connector.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/Connector.java
index d2e7321,0000000..3189d44
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/Connector.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/Connector.java
@@@ -1,210 -1,0 +1,208 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
+import org.apache.accumulo.core.client.admin.InstanceOperations;
+import org.apache.accumulo.core.client.admin.SecurityOperations;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.security.Authorizations;
+
+/**
+ * Connector connects to an Accumulo instance and allows the user to request readers and writers for the instance as well as various objects that permit
+ * administrative operations.
+ *
+ * The Connector enforces security on the client side by forcing all API calls to be accompanied by user credentials.
+ */
+public abstract class Connector {
+
+ /**
+ * Factory method to create a BatchScanner connected to Accumulo.
+ *
+ * @param tableName
+ * the name of the table to query
+ * @param authorizations
+ * A set of authorization labels that will be checked against the column visibility of each key in order to filter data. The authorizations passed in
+ * must be a subset of the accumulo user's set of authorizations. If the accumulo user has authorizations (A1, A2) and authorizations (A2, A3) are
+ * passed, then an exception will be thrown.
+ * @param numQueryThreads
+ * the number of concurrent threads to spawn for querying
+ *
+ * @return BatchScanner object for configuring and querying
+ * @throws TableNotFoundException
+ * when the specified table doesn't exist
+ */
+ public abstract BatchScanner createBatchScanner(String tableName, Authorizations authorizations, int numQueryThreads) throws TableNotFoundException;
+
+ /**
+ * Factory method to create a BatchDeleter connected to Accumulo.
+ *
+ * @param tableName
+ * the name of the table to query and delete from
+ * @param authorizations
+ * A set of authorization labels that will be checked against the column visibility of each key in order to filter data. The authorizations passed in
+ * must be a subset of the accumulo user's set of authorizations. If the accumulo user has authorizations (A1, A2) and authorizations (A2, A3) are
+ * passed, then an exception will be thrown.
+ * @param numQueryThreads
+ * the number of concurrent threads to spawn for querying
+ * @param maxMemory
+ * size in bytes of the maximum memory to batch before writing
+ * @param maxLatency
+ * size in milliseconds; set to 0 or Long.MAX_VALUE to allow the maximum time to hold a batch before writing
+ * @param maxWriteThreads
+ * the maximum number of threads to use for writing data to the tablet servers
+ *
+ * @return BatchDeleter object for configuring and deleting
+ * @throws TableNotFoundException
+ * when the specified table doesn't exist
+ * @deprecated since 1.5.0; Use {@link #createBatchDeleter(String, Authorizations, int, BatchWriterConfig)} instead.
+ */
+ @Deprecated
+ public abstract BatchDeleter createBatchDeleter(String tableName, Authorizations authorizations, int numQueryThreads, long maxMemory, long maxLatency,
+ int maxWriteThreads) throws TableNotFoundException;
+
+ /**
+ *
+ * @param tableName
+ * the name of the table to query and delete from
+ * @param authorizations
+ * A set of authorization labels that will be checked against the column visibility of each key in order to filter data. The authorizations passed in
+ * must be a subset of the accumulo user's set of authorizations. If the accumulo user has authorizations (A1, A2) and authorizations (A2, A3) are
+ * passed, then an exception will be thrown.
+ * @param numQueryThreads
+ * the number of concurrent threads to spawn for querying
+ * @param config
+ * configuration used to create batch writer
+ * @return BatchDeleter object for configuring and deleting
- * @throws TableNotFoundException
+ * @since 1.5.0
+ */
+
+ public abstract BatchDeleter createBatchDeleter(String tableName, Authorizations authorizations, int numQueryThreads, BatchWriterConfig config)
+ throws TableNotFoundException;
+
+ /**
+ * Factory method to create a BatchWriter connected to Accumulo.
+ *
+ * @param tableName
+ * the name of the table to insert data into
+ * @param maxMemory
+ * size in bytes of the maximum memory to batch before writing
+ * @param maxLatency
+ * time in milliseconds; set to 0 or Long.MAX_VALUE to allow the maximum time to hold a batch before writing
+ * @param maxWriteThreads
+ * the maximum number of threads to use for writing data to the tablet servers
+ *
+ * @return BatchWriter object for configuring and writing data to
+ * @throws TableNotFoundException
+ * when the specified table doesn't exist
+ * @deprecated since 1.5.0; Use {@link #createBatchWriter(String, BatchWriterConfig)} instead.
+ */
+ @Deprecated
+ public abstract BatchWriter createBatchWriter(String tableName, long maxMemory, long maxLatency, int maxWriteThreads) throws TableNotFoundException;
+
+ /**
+ * Factory method to create a BatchWriter connected to Accumulo.
+ *
+ * @param tableName
+ * the name of the table to insert data into
+ * @param config
+ * configuration used to create batch writer
+ * @return BatchWriter object for configuring and writing data to
- * @throws TableNotFoundException
+ * @since 1.5.0
+ */
+
+ public abstract BatchWriter createBatchWriter(String tableName, BatchWriterConfig config) throws TableNotFoundException;
+
+ /**
+ * Factory method to create a Multi-Table BatchWriter connected to Accumulo. Multi-table batch writers can queue data for multiple tables, which is good for
+ * ingesting data into multiple tables from the same source
+ *
+ * @param maxMemory
+ * size in bytes of the maximum memory to batch before writing
+ * @param maxLatency
+ * size in milliseconds; set to 0 or Long.MAX_VALUE to allow the maximum time to hold a batch before writing
+ * @param maxWriteThreads
+ * the maximum number of threads to use for writing data to the tablet servers
+ *
+ * @return MultiTableBatchWriter object for configuring and writing data to
+ * @deprecated since 1.5.0; Use {@link #createMultiTableBatchWriter(BatchWriterConfig)} instead.
+ */
+ @Deprecated
+ public abstract MultiTableBatchWriter createMultiTableBatchWriter(long maxMemory, long maxLatency, int maxWriteThreads);
+
+ /**
+ * Factory method to create a Multi-Table BatchWriter connected to Accumulo. Multi-table batch writers can queue data for multiple tables. Also data for
+ * multiple tables can be sent to a server in a single batch. Its an efficient way to ingest data into multiple tables from a single process.
+ *
+ * @param config
+ * configuration used to create multi-table batch writer
+ * @return MultiTableBatchWriter object for configuring and writing data to
+ * @since 1.5.0
+ */
+
+ public abstract MultiTableBatchWriter createMultiTableBatchWriter(BatchWriterConfig config);
+
+ /**
+ * Factory method to create a Scanner connected to Accumulo.
+ *
+ * @param tableName
+ * the name of the table to query data from
+ * @param authorizations
+ * A set of authorization labels that will be checked against the column visibility of each key in order to filter data. The authorizations passed in
+ * must be a subset of the accumulo user's set of authorizations. If the accumulo user has authorizations (A1, A2) and authorizations (A2, A3) are
+ * passed, then an exception will be thrown.
+ *
+ * @return Scanner object for configuring and querying data with
+ * @throws TableNotFoundException
+ * when the specified table doesn't exist
+ */
+ public abstract Scanner createScanner(String tableName, Authorizations authorizations) throws TableNotFoundException;
+
+ /**
+ * Accessor method for internal instance object.
+ *
+ * @return the internal instance object
+ */
+ public abstract Instance getInstance();
+
+ /**
+ * Get the current user for this connector
+ *
+ * @return the user name
+ */
+ public abstract String whoami();
+
+ /**
+ * Retrieves a TableOperations object to perform table functions, such as create and delete.
+ *
+ * @return an object to manipulate tables
+ */
+ public abstract TableOperations tableOperations();
+
+ /**
+ * Retrieves a SecurityOperations object to perform user security operations, such as creating users.
+ *
+ * @return an object to modify users and permissions
+ */
+ public abstract SecurityOperations securityOperations();
+
+ /**
+ * Retrieves an InstanceOperations object to modify instance configuration.
+ *
+ * @return an object to modify instance configuration
+ */
+ public abstract InstanceOperations instanceOperations();
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
index 85e996a,0000000..e58a1be
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
@@@ -1,387 -1,0 +1,383 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+
+/**
+ * Configure an iterator for minc, majc, and/or scan. By default, IteratorSetting will be configured for scan.
+ *
+ * Every iterator has a priority, a name, a class, a set of scopes, and configuration parameters.
+ *
+ * A typical use case configured for scan:
+ *
+ * <pre>
+ * IteratorSetting cfg = new IteratorSetting(priority, "myIter", MyIterator.class);
+ * MyIterator.addOption(cfg, 42);
+ * scanner.addScanIterator(cfg);
+ * </pre>
+ */
+public class IteratorSetting implements Writable {
+ private int priority;
+ private String name;
+ private String iteratorClass;
+ private Map<String,String> properties;
+
+ /**
+ * Get layer at which this iterator applies. See {@link #setPriority(int)} for how the priority is used.
+ *
+ * @return the priority of this Iterator
+ */
+ public int getPriority() {
+ return priority;
+ }
+
+ /**
+ * Set layer at which this iterator applies.
+ *
+ * @param priority
+ * determines the order in which iterators are applied (system iterators are always applied first, then user-configured iterators, lowest priority
+ * first)
+ */
+ public void setPriority(int priority) {
+ ArgumentChecker.strictlyPositive(priority);
+ this.priority = priority;
+ }
+
+ /**
+ * Get the iterator's name.
+ *
+ * @return the name of the iterator
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Set the iterator's name. Must be a simple alphanumeric identifier.
- *
- * @param name
+ */
+ public void setName(String name) {
+ ArgumentChecker.notNull(name);
+ this.name = name;
+ }
+
+ /**
+ * Get the name of the class that implements the iterator.
+ *
+ * @return the iterator's class name
+ */
+ public String getIteratorClass() {
+ return iteratorClass;
+ }
+
+ /**
+ * Set the name of the class that implements the iterator. The class does not have to be present on the client, but it must be available to all tablet
+ * servers.
- *
- * @param iteratorClass
+ */
+ public void setIteratorClass(String iteratorClass) {
+ ArgumentChecker.notNull(iteratorClass);
+ this.iteratorClass = iteratorClass;
+ }
+
+ /**
+ * Constructs an iterator setting configured for the scan scope with no parameters. (Parameters can be added later.)
+ *
+ * @param priority
+ * the priority for the iterator (see {@link #setPriority(int)})
+ * @param name
+ * the distinguishing name for the iterator
+ * @param iteratorClass
+ * the fully qualified class name for the iterator
+ */
+ public IteratorSetting(int priority, String name, String iteratorClass) {
+ this(priority, name, iteratorClass, new HashMap<String,String>());
+ }
+
+ /**
+ * Constructs an iterator setting configured for the specified scopes with the specified parameters.
+ *
+ * @param priority
+ * the priority for the iterator (see {@link #setPriority(int)})
+ * @param name
+ * the distinguishing name for the iterator
+ * @param iteratorClass
+ * the fully qualified class name for the iterator
+ * @param properties
+ * any properties for the iterator
+ */
+ public IteratorSetting(int priority, String name, String iteratorClass, Map<String,String> properties) {
+ setPriority(priority);
+ setName(name);
+ setIteratorClass(iteratorClass);
+ this.properties = new HashMap<String,String>();
+ addOptions(properties);
+ }
+
+ /**
+ * Constructs an iterator setting using the given class's SimpleName for the iterator name. The iterator setting will be configured for the scan scope with no
+ * parameters.
+ *
+ * @param priority
+ * the priority for the iterator (see {@link #setPriority(int)})
+ * @param iteratorClass
+ * the class for the iterator
+ */
+ public IteratorSetting(int priority, Class<? extends SortedKeyValueIterator<Key,Value>> iteratorClass) {
+ this(priority, iteratorClass.getSimpleName(), iteratorClass.getName());
+ }
+
+ /**
+ *
+ * Constructs an iterator setting using the given class's SimpleName for the iterator name and configured for the specified scopes with the specified
+ * parameters.
+ *
+ * @param priority
+ * the priority for the iterator (see {@link #setPriority(int)})
+ * @param iteratorClass
+ * the class for the iterator
+ * @param properties
+ * any properties for the iterator
+ */
+ public IteratorSetting(int priority, Class<? extends SortedKeyValueIterator<Key,Value>> iteratorClass, Map<String,String> properties) {
+ this(priority, iteratorClass.getSimpleName(), iteratorClass.getName(), properties);
+ }
+
+ /**
+ * Constructs an iterator setting configured for the scan scope with no parameters.
+ *
+ * @param priority
+ * the priority for the iterator (see {@link #setPriority(int)})
+ * @param name
+ * the distinguishing name for the iterator
+ * @param iteratorClass
+ * the class for the iterator
+ */
+ public IteratorSetting(int priority, String name, Class<? extends SortedKeyValueIterator<Key,Value>> iteratorClass) {
+ this(priority, name, iteratorClass.getName());
+ }
+
+ /**
+ * @since 1.5.0
+ */
+ public IteratorSetting(DataInput din) throws IOException {
+ this.properties = new HashMap<String,String>();
+ this.readFields(din);
+ }
+
+ /**
+ * Add another option to the iterator.
+ *
+ * @param option
+ * the name of the option
+ * @param value
+ * the value of the option
+ */
+ public void addOption(String option, String value) {
+ ArgumentChecker.notNull(option, value);
+ properties.put(option, value);
+ }
+
+ /**
+ * Remove an option from the iterator.
+ *
+ * @param option
+ * the name of the option
+ * @return the value previously associated with the option, or null if no such option existed
+ */
+ public String removeOption(String option) {
+ ArgumentChecker.notNull(option);
+ return properties.remove(option);
+ }
+
+ /**
+ * Add many options to the iterator.
+ *
+ * @param propertyEntries
+ * a set of entries to add to the options
+ */
+ public void addOptions(Set<Entry<String,String>> propertyEntries) {
+ ArgumentChecker.notNull(propertyEntries);
+ for (Entry<String,String> keyValue : propertyEntries) {
+ addOption(keyValue.getKey(), keyValue.getValue());
+ }
+ }
+
+ /**
+ * Add many options to the iterator.
+ *
+ * @param properties
+ * a map of entries to add to the options
+ */
+ public void addOptions(Map<String,String> properties) {
+ ArgumentChecker.notNull(properties);
+ addOptions(properties.entrySet());
+ }
+
+ /**
+ * Get the configuration parameters for this iterator.
+ *
+ * @return the properties
+ */
+ public Map<String,String> getOptions() {
+ return Collections.unmodifiableMap(properties);
+ }
+
+ /**
+ * Remove all options from the iterator.
+ */
+ public void clearOptions() {
+ properties.clear();
+ }
+
+ /**
+ * @see java.lang.Object#hashCode()
+ */
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((iteratorClass == null) ? 0 : iteratorClass.hashCode());
+ result = prime * result + ((name == null) ? 0 : name.hashCode());
+ result = prime * result + priority;
+ result = prime * result + ((properties == null) ? 0 : properties.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (!(obj instanceof IteratorSetting))
+ return false;
+ IteratorSetting other = (IteratorSetting) obj;
+ if (iteratorClass == null) {
+ if (other.iteratorClass != null)
+ return false;
+ } else if (!iteratorClass.equals(other.iteratorClass))
+ return false;
+ if (name == null) {
+ if (other.name != null)
+ return false;
+ } else if (!name.equals(other.name))
+ return false;
+ if (priority != other.priority)
+ return false;
+ if (properties == null) {
+ if (other.properties != null)
+ return false;
+ } else if (!properties.equals(other.properties))
+ return false;
+ return true;
+ }
+
+ /**
+ * @see java.lang.Object#toString()
+ */
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("name:");
+ sb.append(name);
+ sb.append(", priority:");
+ sb.append(Integer.toString(priority));
+ sb.append(", class:");
+ sb.append(iteratorClass);
+ sb.append(", properties:");
+ sb.append(properties);
+ return sb.toString();
+ }
+
+ /**
+ * A convenience class for passing column family and column qualifiers to iterator configuration methods.
+ */
+ public static class Column extends Pair<Text,Text> {
+
+ public Column(Text columnFamily, Text columnQualifier) {
+ super(columnFamily, columnQualifier);
+ }
+
+ public Column(Text columnFamily) {
+ super(columnFamily, null);
+ }
+
+ public Column(String columnFamily, String columnQualifier) {
+ super(new Text(columnFamily), new Text(columnQualifier));
+ }
+
+ public Column(String columnFamily) {
+ super(new Text(columnFamily), null);
+ }
+
+ public Text getColumnFamily() {
+ return getFirst();
+ }
+
+ public Text getColumnQualifier() {
+ return getSecond();
+ }
+
+ }
+
+ /**
+ * @since 1.5.0
+ */
+ @Override
+ public void readFields(DataInput din) throws IOException {
+ priority = WritableUtils.readVInt(din);
+ name = WritableUtils.readString(din);
+ iteratorClass = WritableUtils.readString(din);
+ properties.clear();
+ int size = WritableUtils.readVInt(din);
+ while (size > 0) {
+ properties.put(WritableUtils.readString(din), WritableUtils.readString(din));
+ size--;
+ }
+ }
+
+ /**
+ * @since 1.5.0
+ */
+ @Override
+ public void write(DataOutput dout) throws IOException {
+ WritableUtils.writeVInt(dout, priority);
+ WritableUtils.writeString(dout, name);
+ WritableUtils.writeString(dout, iteratorClass);
+ WritableUtils.writeVInt(dout, properties.size());
+ for (Entry<String,String> e : properties.entrySet()) {
+ WritableUtils.writeString(dout, e.getKey());
+ WritableUtils.writeString(dout, e.getValue());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/RowIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/RowIterator.java
index 005f697,0000000..f5e9547
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/RowIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/RowIterator.java
@@@ -1,164 -1,0 +1,160 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.NoSuchElementException;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.PeekingIterator;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Group Key/Value pairs into Iterators over rows. Suggested usage:
+ *
+ * <pre>
+ * RowIterator rowIterator = new RowIterator(connector.createScanner(tableName, authorizations));
+ * </pre>
+ */
+public class RowIterator implements Iterator<Iterator<Entry<Key,Value>>> {
+
+ /**
+ * Iterate over entries in a single row.
+ */
+ private static class SingleRowIter implements Iterator<Entry<Key,Value>> {
+ private PeekingIterator<Entry<Key,Value>> source;
+ private Text currentRow = null;
+ private long count = 0;
+ private boolean disabled = false;
+
+ /**
+ * SingleRowIter must be passed a PeekingIterator so that it can peek at the next entry to see if it belongs in the current row or not.
+ */
+ public SingleRowIter(PeekingIterator<Entry<Key,Value>> source) {
+ this.source = source;
+ if (source.hasNext())
+ currentRow = source.peek().getKey().getRow();
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (disabled)
+ throw new IllegalStateException("SingleRowIter no longer valid");
+ return currentRow != null;
+ }
+
+ @Override
+ public Entry<Key,Value> next() {
+ if (disabled)
+ throw new IllegalStateException("SingleRowIter no longer valid");
+ return _next();
+ }
+
+ private Entry<Key,Value> _next() {
+ if (currentRow == null)
+ throw new NoSuchElementException();
+ count++;
+ Entry<Key,Value> kv = source.next();
+ if (!source.hasNext() || !source.peek().getKey().getRow().equals(currentRow)) {
+ currentRow = null;
+ }
+ return kv;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Get a count of entries read from the row (only equals the number of entries in the row when the row has been read fully).
+ */
+ public long getCount() {
+ return count;
+ }
+
+ /**
+ * Consume the rest of the row. Disables the iterator from future use.
+ */
+ public void consume() {
+ disabled = true;
+ while (currentRow != null)
+ _next();
+ }
+ }
+
+ private final PeekingIterator<Entry<Key,Value>> iter;
+ private long count = 0;
+ private SingleRowIter lastIter = null;
+
+ /**
+ * Create an iterator from an (ordered) sequence of KeyValue pairs.
- *
- * @param iterator
+ */
+ public RowIterator(Iterator<Entry<Key,Value>> iterator) {
+ this.iter = new PeekingIterator<Entry<Key,Value>>(iterator);
+ }
+
+ /**
+ * Create an iterator from an Iterable.
- *
- * @param iterable
+ */
+ public RowIterator(Iterable<Entry<Key,Value>> iterable) {
+ this(iterable.iterator());
+ }
+
+ /**
+ * Returns true if there is at least one more row to get.
+ *
+ * If the last row hasn't been fully read, this method will read through the end of the last row so it can determine if the underlying iterator has a next
+ * row. The last row is disabled from future use.
+ */
+ @Override
+ public boolean hasNext() {
+ if (lastIter != null) {
+ lastIter.consume();
+ count += lastIter.getCount();
+ lastIter = null;
+ }
+ return iter.hasNext();
+ }
+
+ /**
+ * Fetch the next row.
+ */
+ @Override
+ public Iterator<Entry<Key,Value>> next() {
+ if (!hasNext())
+ throw new NoSuchElementException();
+ return lastIter = new SingleRowIter(iter);
+ }
+
+ /**
+ * Unsupported.
+ */
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Get a count of the total number of entries in all rows read so far.
+ */
+ public long getKVCount() {
+ return count;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
index 873a3ad,0000000..7c61d57
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
@@@ -1,130 -1,0 +1,130 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.hadoop.io.Text;
+
+/**
+ * This class hosts configuration methods that are shared between different types of scanners.
+ *
+ */
+public interface ScannerBase extends Iterable<Entry<Key,Value>> {
+
+ /**
+ * Add a server-side scan iterator.
+ *
+ * @param cfg
+ * fully specified scan-time iterator, including all options for the iterator. Any changes to the iterator setting after this call are not propagated
+ * to the stored iterator.
+ * @throws IllegalArgumentException
+ * if the setting conflicts with existing iterators
+ */
+ public void addScanIterator(IteratorSetting cfg);
+
+ /**
+ * Remove an iterator from the list of iterators.
+ *
+ * @param iteratorName
+ * nickname used for the iterator
+ */
+ public void removeScanIterator(String iteratorName);
+
+ /**
+ * Update the options for an iterator. Note that this does <b>not</b> change the iterator options during a scan, it just replaces the given option on a
+ * configured iterator before a scan is started.
+ *
+ * @param iteratorName
+ * the name of the iterator to change
+ * @param key
+ * the name of the option
+ * @param value
+ * the new value for the named option
+ */
+ public void updateScanIteratorOption(String iteratorName, String key, String value);
+
+ /**
+ * Adds a column family to the list of columns that will be fetched by this scanner. By default when no columns have been added the scanner fetches all
+ * columns.
+ *
+ * @param col
+ * the column family to be fetched
+ */
+ public void fetchColumnFamily(Text col);
+
+ /**
+ * Adds a column to the list of columns that will be fetched by this scanner. The column is identified by family and qualifier. By default when no columns
+ * have been added the scanner fetches all columns.
+ *
+ * @param colFam
+ * the column family of the column to be fetched
+ * @param colQual
+ * the column qualifier of the column to be fetched
+ */
+ public void fetchColumn(Text colFam, Text colQual);
+
+ /**
+ * Clears the columns to be fetched (useful for resetting the scanner for reuse). Once cleared, the scanner will fetch all columns.
+ */
+ public void clearColumns();
+
+ /**
+ * Clears scan iterators prior to returning a scanner to the pool.
+ */
+ public void clearScanIterators();
+
+ /**
+ * Returns an iterator over an accumulo table. This iterator uses the options that are currently set for its lifetime, so setting options will have no effect
+ * on existing iterators.
+ *
+ * Keys returned by the iterator are not guaranteed to be in sorted order.
+ *
+ * @return an iterator over Key,Value pairs which meet the restrictions set on the scanner
+ */
++ @Override
+ public Iterator<Entry<Key,Value>> iterator();
+
+ /**
+ * This setting determines how long a scanner will automatically retry when a failure occurs. By default a scanner will retry forever.
+ *
+ * Setting to zero or Long.MAX_VALUE and TimeUnit.MILLISECONDS means to retry forever.
+ *
- * @param timeOut
+ * @param timeUnit
+ * determines how timeout is interpreted
+ * @since 1.5.0
+ */
+ public void setTimeout(long timeOut, TimeUnit timeUnit);
+
+ /**
+ * Returns the setting for how long a scanner will automatically retry when a failure occurs.
+ *
+ * @return the timeout configured for this scanner
+ * @since 1.5.0
+ */
+ public long getTimeout(TimeUnit timeUnit);
+
+ /**
+ * Closes any underlying connections on the scanner
+ * @since 1.5.0
+ */
+ public void close();
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
index 5197262,0000000..795ce9f
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
@@@ -1,320 -1,0 +1,318 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.impl.ConnectorImpl;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.OpTimer;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * <p>
+ * An implementation of instance that looks in zookeeper to find information needed to connect to an instance of accumulo.
+ *
+ * <p>
+ * The advantage of using zookeeper to obtain information about accumulo is that zookeeper is highly available, very responsive, and supports caching.
+ *
+ * <p>
+ * Because it is possible for multiple instances of accumulo to share a single set of zookeeper servers, all constructors require an accumulo instance name.
+ *
+ * If you do not know the instance names then run accumulo org.apache.accumulo.server.util.ListInstances on an accumulo server.
+ *
+ */
+
+public class ZooKeeperInstance implements Instance {
+
+ private static final Logger log = Logger.getLogger(ZooKeeperInstance.class);
+
+ private String instanceId = null;
+ private String instanceName = null;
+
+ private final ZooCache zooCache;
+
+ private final String zooKeepers;
+
+ private final int zooKeepersSessionTimeOut;
+
+ /**
+ *
+ * @param instanceName
+ * The name of specific accumulo instance. This is set at initialization time.
+ * @param zooKeepers
+ * A comma separated list of zoo keeper server locations. Each location can contain an optional port, of the format host:port.
+ */
+
+ public ZooKeeperInstance(String instanceName, String zooKeepers) {
+ this(instanceName, zooKeepers, (int) AccumuloConfiguration.getDefaultConfiguration().getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT));
+ }
+
+ /**
+ *
+ * @param instanceName
+ * The name of specific accumulo instance. This is set at initialization time.
+ * @param zooKeepers
+ * A comma separated list of zoo keeper server locations. Each location can contain an optional port, of the format host:port.
+ * @param sessionTimeout
+ * zoo keeper session time out in milliseconds.
+ */
+
+ public ZooKeeperInstance(String instanceName, String zooKeepers, int sessionTimeout) {
+ ArgumentChecker.notNull(instanceName, zooKeepers);
+ this.instanceName = instanceName;
+ this.zooKeepers = zooKeepers;
+ this.zooKeepersSessionTimeOut = sessionTimeout;
+ zooCache = ZooCache.getInstance(zooKeepers, sessionTimeout);
+ getInstanceID();
+ }
+
+ /**
+ *
+ * @param instanceId
+ * The UUID that identifies the accumulo instance you want to connect to.
+ * @param zooKeepers
+ * A comma separated list of zoo keeper server locations. Each location can contain an optional port, of the format host:port.
+ */
+
+ public ZooKeeperInstance(UUID instanceId, String zooKeepers) {
+ this(instanceId, zooKeepers, (int) AccumuloConfiguration.getDefaultConfiguration().getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT));
+ }
+
+ /**
+ *
+ * @param instanceId
+ * The UUID that identifies the accumulo instance you want to connect to.
+ * @param zooKeepers
+ * A comma separated list of zoo keeper server locations. Each location can contain an optional port, of the format host:port.
+ * @param sessionTimeout
+ * zoo keeper session time out in milliseconds.
+ */
+
+ public ZooKeeperInstance(UUID instanceId, String zooKeepers, int sessionTimeout) {
+ ArgumentChecker.notNull(instanceId, zooKeepers);
+ this.instanceId = instanceId.toString();
+ this.zooKeepers = zooKeepers;
+ this.zooKeepersSessionTimeOut = sessionTimeout;
+ zooCache = ZooCache.getInstance(zooKeepers, sessionTimeout);
+ }
+
+ @Override
+ public String getInstanceID() {
+ if (instanceId == null) {
+ // want the instance id to be stable for the life of this instance object,
+ // so only get it once
+ String instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + instanceName;
+ byte[] iidb = zooCache.get(instanceNamePath);
+ if (iidb == null) {
+ throw new RuntimeException("Instance name " + instanceName
+ + " does not exist in zookeeper. Run \"accumulo org.apache.accumulo.server.util.ListInstances\" to see a list.");
+ }
+ instanceId = new String(iidb, Constants.UTF8);
+ }
+
+ if (zooCache.get(Constants.ZROOT + "/" + instanceId) == null) {
+ if (instanceName == null)
+ throw new RuntimeException("Instance id " + instanceId + " does not exist in zookeeper");
+ throw new RuntimeException("Instance id " + instanceId + " pointed to by the name " + instanceName + " does not exist in zookeeper");
+ }
+
+ return instanceId;
+ }
+
+ @Override
+ public List<String> getMasterLocations() {
+ String masterLocPath = ZooUtil.getRoot(this) + Constants.ZMASTER_LOCK;
+
+ OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Looking up master location in zoocache.");
+ byte[] loc = ZooUtil.getLockData(zooCache, masterLocPath);
+ opTimer.stop("Found master at " + (loc == null ? null : new String(loc, Constants.UTF8)) + " in %DURATION%");
+
+ if (loc == null) {
+ return Collections.emptyList();
+ }
+
+ return Collections.singletonList(new String(loc, Constants.UTF8));
+ }
+
+ @Override
+ public String getRootTabletLocation() {
+ String zRootLocPath = ZooUtil.getRoot(this) + Constants.ZROOT_TABLET_LOCATION;
+
+ OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Looking up root tablet location in zookeeper.");
+ byte[] loc = zooCache.get(zRootLocPath);
+ opTimer.stop("Found root tablet at " + (loc == null ? null : new String(loc, Constants.UTF8)) + " in %DURATION%");
+
+ if (loc == null) {
+ return null;
+ }
+
+ return new String(loc, Constants.UTF8).split("\\|")[0];
+ }
+
+ @Override
+ public String getInstanceName() {
+ if (instanceName == null)
+ instanceName = lookupInstanceName(zooCache, UUID.fromString(getInstanceID()));
+
+ return instanceName;
+ }
+
+ @Override
+ public String getZooKeepers() {
+ return zooKeepers;
+ }
+
+ @Override
+ public int getZooKeepersSessionTimeOut() {
+ return zooKeepersSessionTimeOut;
+ }
+
+ @Override
+ @Deprecated
+ public Connector getConnector(String user, CharSequence pass) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(user, TextUtil.getBytes(new Text(pass.toString())));
+ }
+
+ @Override
+ @Deprecated
+ public Connector getConnector(String user, ByteBuffer pass) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(user, ByteBufferUtil.toBytes(pass));
+ }
+
+ @Override
+ public Connector getConnector(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(CredentialHelper.create(principal, token, getInstanceID()));
+ }
+
+ @SuppressWarnings("deprecation")
+ private Connector getConnector(TCredentials credential) throws AccumuloException, AccumuloSecurityException {
+ return new ConnectorImpl(this, credential);
+ }
+
+ @Override
+ @Deprecated
+ public Connector getConnector(String principal, byte[] pass) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(principal, new PasswordToken(pass));
+ }
+
+ private AccumuloConfiguration conf = null;
+
+ @Override
+ public AccumuloConfiguration getConfiguration() {
+ if (conf == null)
+ conf = AccumuloConfiguration.getDefaultConfiguration();
+ return conf;
+ }
+
+ @Override
+ public void setConfiguration(AccumuloConfiguration conf) {
+ this.conf = conf;
+ }
+
+ /**
+ * @deprecated Use {@link #lookupInstanceName(org.apache.accumulo.fate.zookeeper.ZooCache, UUID)} instead
+ */
+ @Deprecated
+ public static String lookupInstanceName(org.apache.accumulo.core.zookeeper.ZooCache zooCache, UUID instanceId) {
+ return lookupInstanceName((ZooCache) zooCache, instanceId);
+ }
+
+ /**
+ * Given a zooCache and instanceId, look up the instance name.
+ *
- * @param zooCache
- * @param instanceId
+ * @return the instance name
+ */
+ public static String lookupInstanceName(ZooCache zooCache, UUID instanceId) {
+ ArgumentChecker.notNull(zooCache, instanceId);
+ for (String name : zooCache.getChildren(Constants.ZROOT + Constants.ZINSTANCES)) {
+ String instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + name;
+ byte[] bytes = zooCache.get(instanceNamePath);
+ UUID iid = UUID.fromString(new String(bytes, Constants.UTF8));
+ if (iid.equals(instanceId)) {
+ return name;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * To be moved to server code. Only lives here to support certain client side utilities to minimize command-line options.
+ */
+ @Deprecated
+ public static String getInstanceIDFromHdfs(Path instanceDirectory) {
+ try {
+ FileSystem fs = FileUtil.getFileSystem(CachedConfiguration.getInstance(), AccumuloConfiguration.getSiteConfiguration());
+ FileStatus[] files = null;
+ try {
+ files = fs.listStatus(instanceDirectory);
+ } catch (FileNotFoundException ex) {
+ // ignored
+ }
+ log.debug("Trying to read instance id from " + instanceDirectory);
+ if (files == null || files.length == 0) {
+ log.error("unable obtain instance id at " + instanceDirectory);
+ throw new RuntimeException("Accumulo not initialized, there is no instance id at " + instanceDirectory);
+ } else if (files.length != 1) {
+ log.error("multiple potential instances in " + instanceDirectory);
+ throw new RuntimeException("Accumulo found multiple possible instance ids in " + instanceDirectory);
+ } else {
+ String result = files[0].getPath().getName();
+ return result;
+ }
+ } catch (IOException e) {
+ log.error("Problem reading instance id out of hdfs at " + instanceDirectory, e);
+ throw new RuntimeException("Can't tell if Accumulo is initialized; can't read instance id at " + instanceDirectory, e);
+ } catch (IllegalArgumentException exception) {
+ /* HDFS throws this when there's a UnknownHostException due to DNS troubles. */
+ if (exception.getCause() instanceof UnknownHostException) {
+ log.error("Problem reading instance id out of hdfs at " + instanceDirectory, exception);
+ }
+ throw exception;
+ }
+ }
+
+ @Deprecated
+ @Override
+ public Connector getConnector(org.apache.accumulo.core.security.thrift.AuthInfo auth) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(auth.user, auth.password);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/admin/ActiveCompaction.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/admin/ActiveCompaction.java
index f2876b2,0000000..9c39ea6
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/ActiveCompaction.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/ActiveCompaction.java
@@@ -1,185 -1,0 +1,184 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.admin;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+
+
+/**
+ *
+ * @since 1.5.0
+ */
+public class ActiveCompaction {
+
+ private org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction tac;
+ private Instance instance;
+
+ ActiveCompaction(Instance instance, org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction tac) {
+ this.tac = tac;
+ this.instance = instance;
+ }
+
+ public static enum CompactionType {
+ /**
+ * compaction to flush a tablets memory
+ */
+ MINOR,
+ /**
+ * compaction to flush a tablets memory and merge it with the tablets smallest file. This type compaction is done when a tablet has too many files
+ */
+ MERGE,
+ /**
+ * compaction that merges a subset of a tablets files into one file
+ */
+ MAJOR,
+ /**
+ * compaction that merges all of a tablets files into one file
+ */
+ FULL
+ };
+
+ public static enum CompactionReason {
+ /**
+ * compaction initiated by user
+ */
+ USER,
+ /**
+ * Compaction initiated by system
+ */
+ SYSTEM,
+ /**
+ * Compaction initiated by merge operation
+ */
+ CHOP,
+ /**
+ * idle compaction
+ */
+ IDLE,
+ /**
+ * Compaction initiated to close a unload a tablet
+ */
+ CLOSE
+ };
+
+ /**
+ *
+ * @return name of the table the compaction is running against
- * @throws TableNotFoundException
+ */
+
+ public String getTable() throws TableNotFoundException {
+ return Tables.getTableName(instance, getExtent().getTableId().toString());
+ }
+
+ /**
+ * @return tablet thats is compacting
+ */
+
+ public KeyExtent getExtent() {
+ return new KeyExtent(tac.getExtent());
+ }
+
+ /**
+ * @return how long the compaction has been running in milliseconds
+ */
+
+ public long getAge() {
+ return tac.getAge();
+ }
+
+ /**
+ * @return the files the compaction is reading from
+ */
+
+ public List<String> getInputFiles() {
+ return tac.getInputFiles();
+ }
+
+ /**
+ * @return file compactions is writing too
+ */
+
+ public String getOutputFile() {
+ return tac.getOutputFile();
+ }
+
+ /**
+ * @return the type of compaction
+ */
+ public CompactionType getType() {
+ return CompactionType.valueOf(tac.getType().name());
+ }
+
+ /**
+ * @return the reason the compaction was started
+ */
+
+ public CompactionReason getReason() {
+ return CompactionReason.valueOf(tac.getReason().name());
+ }
+
+ /**
+ * @return the locality group that is compacting
+ */
+
+ public String getLocalityGroup() {
+ return tac.getLocalityGroup();
+ }
+
+ /**
+ * @return the number of key/values read by the compaction
+ */
+
+ public long getEntriesRead() {
+ return tac.getEntriesRead();
+ }
+
+ /**
+ * @return the number of key/values written by the compaction
+ */
+
+ public long getEntriesWritten() {
+ return tac.getEntriesWritten();
+ }
+
+ /**
+ * @return the per compaction iterators configured
+ */
+
+ public List<IteratorSetting> getIterators() {
+ ArrayList<IteratorSetting> ret = new ArrayList<IteratorSetting>();
+
+ for (IterInfo ii : tac.getSsiList()) {
+ IteratorSetting settings = new IteratorSetting(ii.getPriority(), ii.getIterName(), ii.getClassName());
+ Map<String,String> options = tac.getSsio().get(ii.getIterName());
+ settings.addOptions(options);
+
+ ret.add(settings);
+ }
+
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
index fce0716,0000000..29ff2a6
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
@@@ -1,131 -1,0 +1,119 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.admin;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+
+/**
+ *
+ */
+public interface InstanceOperations {
+
+ /**
+ * Sets an system property in zookeeper. Tablet servers will pull this setting and override the equivalent setting in accumulo-site.xml. Changes can be seen
+ * using {@link #getSystemConfiguration()}
+ *
+ * @param property
+ * the name of a per-table property
+ * @param value
+ * the value to set a per-table property to
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ public void setProperty(final String property, final String value) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Removes a system property from zookeeper. Changes can be seen using {@link #getSystemConfiguration()}
+ *
+ * @param property
+ * the name of a per-table property
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ public void removeProperty(final String property) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ *
+ * @return A map of system properties set in zookeeper. If a property is not set in zookeeper, then it will return the value set in accumulo-site.xml on some
+ * server. If nothing is set in an accumulo-site.xml file it will return the default value for each property.
- * @throws AccumuloException
- * @throws AccumuloSecurityException
+ */
+
+ public Map<String,String> getSystemConfiguration() throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ *
+ * @return A map of system properties set in accumulo-site.xml on some server. If nothing is set in an accumulo-site.xml file it will return the default value
+ * for each property.
- * @throws AccumuloException
- * @throws AccumuloSecurityException
+ */
+
+ public Map<String,String> getSiteConfiguration() throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * List the currently active tablet servers participating in the accumulo instance
+ *
+ * @return A list of currently active tablet servers.
+ */
+
+ public List<String> getTabletServers();
+
+ /**
+ * List the active scans on tablet server.
+ *
+ * @param tserver
+ * The tablet server address should be of the form <ip address>:<port>
+ * @return A list of active scans on tablet server.
- * @throws AccumuloException
- * @throws AccumuloSecurityException
+ */
+
+ public List<ActiveScan> getActiveScans(String tserver) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * List the active compaction running on a tablet server
+ *
+ * @param tserver
+ * The tablet server address should be of the form <ip address>:<port>
+ * @return the list of active compactions
- * @throws AccumuloException
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+
+ public List<ActiveCompaction> getActiveCompactions(String tserver) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Throws an exception if a tablet server can not be contacted.
+ *
+ * @param tserver
+ * The tablet server address should be of the form <ip address>:<port>
- * @throws AccumuloException
+ * @since 1.5.0
+ */
+ public void ping(String tserver) throws AccumuloException;
+
+ /**
+ * Test to see if the instance can load the given class as the given type. This check does not consider per table classpaths, see
+ * {@link TableOperations#testClassLoad(String, String, String)}
+ *
- * @param className
- * @param asTypeName
+ * @return true if the instance can load the given class as the given type, false otherwise
- * @throws AccumuloException
+ */
+ public boolean testClassLoad(final String className, final String asTypeName) throws AccumuloException, AccumuloSecurityException;
+
+}
[08/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
index 8835b1c,0000000..53ea0e8
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
@@@ -1,676 -1,0 +1,672 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators.user;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.NoSuchElementException;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.OptionDescriber;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.WrappingIterator;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.VisibilityEvaluator;
+import org.apache.accumulo.core.security.VisibilityParseException;
+import org.apache.accumulo.core.util.BadArgumentException;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.commons.collections.BufferOverflowException;
+import org.apache.commons.collections.map.LRUMap;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+/**
+ * The TransformingIterator allows portions of a key (except for the row) to be transformed. This iterator handles the details that come with modifying keys
+ * (i.e., that the sort order could change). In order to do so, however, the iterator must put all keys sharing the same prefix in memory. Prefix is defined as
+ * the parts of the key that are not modified by this iterator. That is, if the iterator modifies column qualifier and timestamp, then the prefix is row and
+ * column family. In that case, the iterator must load all column qualifiers for each row/column family pair into memory. Given this constraint, care must be
+ * taken by users of this iterator to ensure it is not run in such a way that will overrun memory in a tablet server.
+ * <p>
+ * If the implementing iterator is transforming column families, then it must also override {@code untransformColumnFamilies(Collection)} to handle the case
+ * when column families are fetched at scan time. The fetched column families will/must be in the transformed space, and the untransformed column families need
+ * to be passed to this iterator's source. If it is not possible to write a reverse transformation (e.g., the column family transformation depends on the row
+ * value or something like that), then the iterator must not fetch specific column families (or only fetch column families that are known to not transform at
+ * all).
+ * <p>
+ * If the implementing iterator is transforming column visibilities, then users must be careful NOT to fetch column qualifiers from the scanner. The reason for
+ * this is due to ACCUMULO-??? (insert issue number).
+ * <p>
+ * If the implementing iterator is transforming column visibilities, then the user should be sure to supply authorizations via the {@link #AUTH_OPT} iterator
+ * option (note that this is only necessary for scan scope iterators). The supplied authorizations should be in the transformed space, but the authorizations
+ * supplied to the scanner should be in the untransformed space. That is, if the iterator transforms A to 1, B to 2, C to 3, etc, then the auths supplied when
+ * the scanner is constructed should be A,B,C,... and the auths supplied to the iterator should be 1,2,3,... The reason for this is that the scanner performs
+ * security filtering before this iterator is called, so the authorizations need to be in the original untransformed space. Since the iterator can transform
+ * visibilities, it is possible that it could produce visibilities that the user cannot see, so the transformed keys must be tested to ensure the user is
+ * allowed to view them. Note that this test is not necessary when the iterator is not used in the scan scope since no security filtering is performed during
+ * major and minor compactions. It should also be noted that this iterator implements the security filtering rather than relying on a follow-on iterator to do
+ * it so that we ensure the test is performed.
+ */
+abstract public class TransformingIterator extends WrappingIterator implements OptionDescriber {
+ public static final String AUTH_OPT = "authorizations";
+ public static final String MAX_BUFFER_SIZE_OPT = "maxBufferSize";
+ private static final long DEFAULT_MAX_BUFFER_SIZE = 10000000;
-
++
+ protected Logger log = Logger.getLogger(getClass());
-
++
+ protected ArrayList<Pair<Key,Value>> keys = new ArrayList<Pair<Key,Value>>();
+ protected int keyPos = -1;
+ protected boolean scanning;
+ protected Range seekRange;
+ protected Collection<ByteSequence> seekColumnFamilies;
+ protected boolean seekColumnFamiliesInclusive;
-
++
+ private VisibilityEvaluator ve = null;
+ private LRUMap visibleCache = null;
+ private LRUMap parsedVisibilitiesCache = null;
+ private long maxBufferSize;
-
++
+ private static Comparator<Pair<Key,Value>> keyComparator = new Comparator<Pair<Key,Value>>() {
+ @Override
+ public int compare(Pair<Key,Value> o1, Pair<Key,Value> o2) {
+ return o1.getFirst().compareTo(o2.getFirst());
+ }
+ };
-
++
+ public TransformingIterator() {}
-
++
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ super.init(source, options, env);
+ scanning = IteratorScope.scan.equals(env.getIteratorScope());
+ if (scanning) {
+ String auths = options.get(AUTH_OPT);
+ if (auths != null && !auths.isEmpty()) {
+ ve = new VisibilityEvaluator(new Authorizations(auths.getBytes(Constants.UTF8)));
+ visibleCache = new LRUMap(100);
+ }
+ }
-
++
+ if (options.containsKey(MAX_BUFFER_SIZE_OPT)) {
+ maxBufferSize = AccumuloConfiguration.getMemoryInBytes(options.get(MAX_BUFFER_SIZE_OPT));
+ } else {
+ maxBufferSize = DEFAULT_MAX_BUFFER_SIZE;
+ }
-
++
+ parsedVisibilitiesCache = new LRUMap(100);
+ }
-
++
+ @Override
+ public IteratorOptions describeOptions() {
+ String desc = "This iterator allows ranges of key to be transformed (with the exception of row transformations).";
+ String authDesc = "Comma-separated list of user's scan authorizations. "
+ + "If excluded or empty, then no visibility check is performed on transformed keys.";
+ String bufferDesc = "Maximum buffer size (in accumulo memory spec) to use for buffering keys before throwing a BufferOverflowException. "
+ + "Users should keep this limit in mind when deciding what to transform. That is, if transforming the column family for example, then all "
+ + "keys sharing the same row and column family must fit within this limit (along with their associated values)";
+ HashMap<String,String> namedOptions = new HashMap<String,String>();
+ namedOptions.put(AUTH_OPT, authDesc);
+ namedOptions.put(MAX_BUFFER_SIZE_OPT, bufferDesc);
+ return new IteratorOptions(getClass().getSimpleName(), desc, namedOptions, null);
+ }
-
++
+ @Override
+ public boolean validateOptions(Map<String,String> options) {
-
++
+ for (Entry<String,String> option : options.entrySet()) {
+ try {
+ if (option.getKey().equals(AUTH_OPT)) {
+ new Authorizations(option.getValue().getBytes(Constants.UTF8));
+ } else if (option.getKey().equals(MAX_BUFFER_SIZE_OPT)) {
+ AccumuloConfiguration.getMemoryInBytes(option.getValue());
+ }
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Failed to parse opt " + option.getKey() + " " + option.getValue(), e);
+ }
+ }
-
++
+ return true;
+ }
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ TransformingIterator copy;
-
++
+ try {
+ copy = getClass().newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
-
++
+ copy.setSource(getSource().deepCopy(env));
-
++
+ copy.scanning = scanning;
+ copy.keyPos = keyPos;
+ copy.keys.addAll(keys);
+ copy.seekRange = (seekRange == null) ? null : new Range(seekRange);
+ copy.seekColumnFamilies = (seekColumnFamilies == null) ? null : new HashSet<ByteSequence>(seekColumnFamilies);
+ copy.seekColumnFamiliesInclusive = seekColumnFamiliesInclusive;
-
++
+ copy.ve = ve;
+ if (visibleCache != null) {
+ copy.visibleCache = new LRUMap(visibleCache.maxSize());
+ copy.visibleCache.putAll(visibleCache);
+ }
-
++
+ if (parsedVisibilitiesCache != null) {
+ copy.parsedVisibilitiesCache = new LRUMap(parsedVisibilitiesCache.maxSize());
+ copy.parsedVisibilitiesCache.putAll(parsedVisibilitiesCache);
+ }
-
++
+ copy.maxBufferSize = maxBufferSize;
-
++
+ return copy;
+ }
-
++
+ @Override
+ public boolean hasTop() {
+ return keyPos >= 0 && keyPos < keys.size();
+ }
-
++
+ @Override
+ public Key getTopKey() {
+ return hasTop() ? keys.get(keyPos).getFirst() : null;
+ }
-
++
+ @Override
+ public Value getTopValue() {
+ return hasTop() ? keys.get(keyPos).getSecond() : null;
+ }
-
++
+ @Override
+ public void next() throws IOException {
+ // Move on to the next entry since we returned the entry at keyPos before
+ if (keyPos >= 0)
+ keyPos++;
-
++
+ // If we emptied out the transformed key map then transform the next key
+ // set from the source. It’s possible that transformation could produce keys
+ // that are outside of our range or are not visible to the end user, so after the
+ // call below we might not have added any keys to the map. Keep going until
+ // we either get some keys in the map or exhaust the source iterator.
+ while (!hasTop() && super.hasTop())
+ transformKeys();
+ }
-
++
+ @Override
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
+ seekRange = new Range(range);
+ seekColumnFamilies = columnFamilies;
+ seekColumnFamiliesInclusive = inclusive;
-
++
+ // Seek the source iterator, but use a recalculated range that ensures
+ // we see all keys with the same "prefix." We need to do this since
+ // transforming could change the sort order and transformed keys that
+ // are before the range start could be inside the range after transformation.
+ super.seek(computeReseekRange(range), untransformColumnFamilies(columnFamilies), inclusive);
-
++
+ // Range clipping could cause us to trim out all the keys we transformed.
+ // Keep looping until we either have some keys in the output range, or have
+ // exhausted the source iterator.
+ keyPos = -1; // “Clear” list so hasTop returns false to get us into the loop (transformKeys actually clears)
+ while (!hasTop() && super.hasTop()) {
+ // Build up a sorted list of all keys for the same prefix. When
+ // people ask for keys, return from this list first until it is empty
+ // before incrementing the source iterator.
+ transformKeys();
+ }
+ }
-
++
+ private static class RangeIterator implements SortedKeyValueIterator<Key,Value> {
-
++
+ private SortedKeyValueIterator<Key,Value> source;
+ private Key prefixKey;
+ private PartialKey keyPrefix;
+ private boolean hasTop = false;
-
++
+ RangeIterator(SortedKeyValueIterator<Key,Value> source, Key prefixKey, PartialKey keyPrefix) {
+ this.source = source;
+ this.prefixKey = prefixKey;
+ this.keyPrefix = keyPrefix;
+ }
-
++
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ throw new UnsupportedOperationException();
+ }
-
++
+ @Override
+ public boolean hasTop() {
+ // only have a top if the prefix matches
+ return hasTop = source.hasTop() && source.getTopKey().equals(prefixKey, keyPrefix);
+ }
-
++
+ @Override
+ public void next() throws IOException {
+ // do not let user advance too far and try to avoid reexecuting hasTop()
+ if (!hasTop && !hasTop())
+ throw new NoSuchElementException();
+ hasTop = false;
+ source.next();
+ }
-
++
+ @Override
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
+ throw new UnsupportedOperationException();
+ }
-
++
+ @Override
+ public Key getTopKey() {
+ return source.getTopKey();
+ }
-
++
+ @Override
+ public Value getTopValue() {
+ return source.getTopValue();
+ }
-
++
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ throw new UnsupportedOperationException();
+ }
-
++
+ }
-
++
+ /**
+ * Reads all keys matching the first key's prefix from the source iterator, transforms them, and sorts the resulting keys. Transformed keys that fall outside
+ * of our seek range or can't be seen by the user are excluded.
+ */
+ protected void transformKeys() throws IOException {
+ keyPos = -1;
+ keys.clear();
+ final Key prefixKey = super.hasTop() ? new Key(super.getTopKey()) : null;
-
++
+ transformRange(new RangeIterator(getSource(), prefixKey, getKeyPrefix()), new KVBuffer() {
-
++
+ long appened = 0;
-
++
+ @Override
+ public void append(Key key, Value val) {
+ // ensure the key provided by the user has the correct prefix
+ if (!key.equals(prefixKey, getKeyPrefix()))
+ throw new IllegalArgumentException("Key prefixes are not equal " + key + " " + prefixKey);
-
++
+ // Transformation could have produced a key that falls outside
+ // of the seek range, or one that the user cannot see. Check
+ // these before adding it to the output list.
+ if (includeTransformedKey(key)) {
-
++
+ // try to defend against a scan or compaction using all memory in a tablet server
+ if (appened > maxBufferSize)
+ throw new BufferOverflowException("Exceeded buffer size of " + maxBufferSize + ", prefixKey: " + prefixKey);
-
++
+ if (getSource().hasTop() && key == getSource().getTopKey())
+ key = new Key(key);
+ keys.add(new Pair<Key,Value>(key, new Value(val)));
+ appened += (key.getSize() + val.getSize() + 128);
+ }
+ }
+ });
-
++
+ // consume any key in range that user did not consume
+ while (super.hasTop() && super.getTopKey().equals(prefixKey, getKeyPrefix())) {
+ super.next();
+ }
-
++
+ if (!keys.isEmpty()) {
+ Collections.sort(keys, keyComparator);
+ keyPos = 0;
+ }
+ }
-
++
+ /**
+ * Determines whether or not to include {@code transformedKey} in the output. It is possible that transformation could have produced a key that falls outside
+ * of the seek range, a key with a visibility the user can't see, a key with a visibility that doesn't parse, or a key with a column family that wasn't
+ * fetched. We only do some checks (outside the range, user can see) if we're scanning. The range check is not done for major/minor compaction since seek
+ * ranges won't be in our transformed key space and we will never change the row so we can't produce keys that would fall outside the tablet anyway.
- *
++ *
+ * @param transformedKey
+ * the key to check
+ * @return {@code true} if the key should be included and {@code false} if not
+ */
+ protected boolean includeTransformedKey(Key transformedKey) {
+ boolean include = canSee(transformedKey);
+ if (scanning && seekRange != null) {
+ include = include && seekRange.contains(transformedKey);
+ }
+ return include;
+ }
-
++
+ /**
+ * Indicates whether or not the user is able to see {@code key}. If the user has not supplied authorizations, or the iterator is not in the scan scope, then
+ * this method simply returns {@code true}. Otherwise, {@code key}'s column visibility is tested against the user-supplied authorizations, and the test result
+ * is returned. For performance, the test results are cached so that the same visibility is not tested multiple times.
- *
++ *
+ * @param key
+ * the key to test
+ * @return {@code true} if the key is visible or iterator is not scanning, and {@code false} if not
+ */
+ protected boolean canSee(Key key) {
+ // Ensure that the visibility (which could have been transformed) parses. Must always do this check, even if visibility is not evaluated.
+ ByteSequence visibility = key.getColumnVisibilityData();
+ ColumnVisibility colVis = null;
+ Boolean parsed = (Boolean) parsedVisibilitiesCache.get(visibility);
+ if (parsed == null) {
+ try {
+ colVis = new ColumnVisibility(visibility.toArray());
+ parsedVisibilitiesCache.put(visibility, Boolean.TRUE);
+ } catch (BadArgumentException e) {
+ log.error("Parse error after transformation : " + visibility);
+ parsedVisibilitiesCache.put(visibility, Boolean.FALSE);
+ if (scanning) {
+ return false;
+ } else {
+ throw e;
+ }
+ }
+ } else if (!parsed) {
+ if (scanning)
+ return false;
+ else
+ throw new IllegalStateException();
+ }
-
++
+ Boolean visible = canSeeColumnFamily(key);
-
++
+ if (!scanning || !visible || ve == null || visibleCache == null || visibility.length() == 0)
+ return visible;
-
++
+ visible = (Boolean) visibleCache.get(visibility);
+ if (visible == null) {
+ try {
+ if (colVis == null)
+ colVis = new ColumnVisibility(visibility.toArray());
+ visible = ve.evaluate(colVis);
+ visibleCache.put(visibility, visible);
+ } catch (VisibilityParseException e) {
+ log.error("Parse Error", e);
+ visible = Boolean.FALSE;
+ } catch (BadArgumentException e) {
+ log.error("Parse Error", e);
+ visible = Boolean.FALSE;
+ }
+ }
-
++
+ return visible;
+ }
-
++
+ /**
+ * Indicates whether or not {@code key} can be seen, according to the fetched column families for this iterator.
- *
++ *
+ * @param key
+ * the key whose column family is to be tested
+ * @return {@code true} if {@code key}'s column family is one of those fetched in the set passed to our {@link #seek(Range, Collection, boolean)} method
+ */
+ protected boolean canSeeColumnFamily(Key key) {
+ boolean visible = true;
+ if (seekColumnFamilies != null) {
+ ByteSequence columnFamily = key.getColumnFamilyData();
+ if (seekColumnFamiliesInclusive)
+ visible = seekColumnFamilies.contains(columnFamily);
+ else
+ visible = !seekColumnFamilies.contains(columnFamily);
+ }
+ return visible;
+ }
-
++
+ /**
+ * Possibly expand {@code range} to include everything for the key prefix we are working with. That is, if our prefix is ROW_COLFAM, then we need to expand
+ * the range so we're sure to include all entries having the same row and column family as the start/end of the range.
- *
++ *
+ * @param range
+ * the range to expand
+ * @return the modified range
+ */
+ protected Range computeReseekRange(Range range) {
+ Key startKey = range.getStartKey();
+ boolean startKeyInclusive = range.isStartKeyInclusive();
+ // If anything after the prefix is set, then clip the key so we include
+ // everything for the prefix.
+ if (isSetAfterPart(startKey, getKeyPrefix())) {
+ startKey = copyPartialKey(startKey, getKeyPrefix());
+ startKeyInclusive = true;
+ }
+ Key endKey = range.getEndKey();
+ boolean endKeyInclusive = range.isEndKeyInclusive();
+ if (isSetAfterPart(endKey, getKeyPrefix())) {
+ endKey = endKey.followingKey(getKeyPrefix());
+ endKeyInclusive = true;
+ }
+ return new Range(startKey, startKeyInclusive, endKey, endKeyInclusive);
+ }
-
++
+ /**
+ * Indicates whether or not any part of {@code key} excluding {@code part} is set. For example, if part is ROW_COLFAM_COLQUAL, then this method determines
+ * whether or not the column visibility, timestamp, or delete flag is set on {@code key}.
- *
++ *
+ * @param key
+ * the key to check
+ * @param part
+ * the part of the key that doesn't need to be checked (everything after does)
+ * @return {@code true} if anything after {@code part} is set on {@code key}, and {@code false} if not
+ */
+ protected boolean isSetAfterPart(Key key, PartialKey part) {
+ boolean isSet = false;
+ if (key != null) {
+ // Breaks excluded on purpose.
+ switch (part) {
+ case ROW:
+ isSet = isSet || key.getColumnFamilyData().length() > 0;
+ case ROW_COLFAM:
+ isSet = isSet || key.getColumnQualifierData().length() > 0;
+ case ROW_COLFAM_COLQUAL:
+ isSet = isSet || key.getColumnVisibilityData().length() > 0;
+ case ROW_COLFAM_COLQUAL_COLVIS:
+ isSet = isSet || key.getTimestamp() < Long.MAX_VALUE;
+ case ROW_COLFAM_COLQUAL_COLVIS_TIME:
+ isSet = isSet || key.isDeleted();
+ case ROW_COLFAM_COLQUAL_COLVIS_TIME_DEL:
+ break;
+ }
+ }
+ return isSet;
+ }
-
++
+ /**
+ * Creates a copy of {@code key}, copying only the parts of the key specified in {@code part}. For example, if {@code part} is ROW_COLFAM_COLQUAL, then this
+ * method would copy the row, column family, and column qualifier from {@code key} into a new key.
- *
++ *
+ * @param key
+ * the key to copy
+ * @param part
+ * the parts of {@code key} to copy
+ * @return the new key containing {@code part} of {@code key}
+ */
+ protected Key copyPartialKey(Key key, PartialKey part) {
+ Key keyCopy;
+ switch (part) {
+ case ROW:
+ keyCopy = new Key(key.getRow());
+ break;
+ case ROW_COLFAM:
+ keyCopy = new Key(key.getRow(), key.getColumnFamily());
+ break;
+ case ROW_COLFAM_COLQUAL:
+ keyCopy = new Key(key.getRow(), key.getColumnFamily(), key.getColumnQualifier());
+ break;
+ case ROW_COLFAM_COLQUAL_COLVIS:
+ keyCopy = new Key(key.getRow(), key.getColumnFamily(), key.getColumnQualifier(), key.getColumnVisibility());
+ break;
+ case ROW_COLFAM_COLQUAL_COLVIS_TIME:
+ keyCopy = new Key(key.getRow(), key.getColumnFamily(), key.getColumnQualifier(), key.getColumnVisibility(), key.getTimestamp());
+ break;
+ default:
+ throw new IllegalArgumentException("Unsupported key part: " + part);
+ }
+ return keyCopy;
+ }
-
++
+ /**
+ * Make a new key with all parts (including delete flag) coming from {@code originalKey} but use {@code newColFam} as the column family.
+ */
+ protected Key replaceColumnFamily(Key originalKey, Text newColFam) {
+ byte[] row = originalKey.getRowData().toArray();
+ byte[] cf = newColFam.getBytes();
+ byte[] cq = originalKey.getColumnQualifierData().toArray();
+ byte[] cv = originalKey.getColumnVisibilityData().toArray();
+ long timestamp = originalKey.getTimestamp();
+ Key newKey = new Key(row, 0, row.length, cf, 0, newColFam.getLength(), cq, 0, cq.length, cv, 0, cv.length, timestamp);
+ newKey.setDeleted(originalKey.isDeleted());
+ return newKey;
+ }
-
++
+ /**
+ * Make a new key with all parts (including delete flag) coming from {@code originalKey} but use {@code newColQual} as the column qualifier.
+ */
+ protected Key replaceColumnQualifier(Key originalKey, Text newColQual) {
+ byte[] row = originalKey.getRowData().toArray();
+ byte[] cf = originalKey.getColumnFamilyData().toArray();
+ byte[] cq = newColQual.getBytes();
+ byte[] cv = originalKey.getColumnVisibilityData().toArray();
+ long timestamp = originalKey.getTimestamp();
+ Key newKey = new Key(row, 0, row.length, cf, 0, cf.length, cq, 0, newColQual.getLength(), cv, 0, cv.length, timestamp);
+ newKey.setDeleted(originalKey.isDeleted());
+ return newKey;
+ }
-
++
+ /**
+ * Make a new key with all parts (including delete flag) coming from {@code originalKey} but use {@code newColVis} as the column visibility.
+ */
+ protected Key replaceColumnVisibility(Key originalKey, Text newColVis) {
+ byte[] row = originalKey.getRowData().toArray();
+ byte[] cf = originalKey.getColumnFamilyData().toArray();
+ byte[] cq = originalKey.getColumnQualifierData().toArray();
+ byte[] cv = newColVis.getBytes();
+ long timestamp = originalKey.getTimestamp();
+ Key newKey = new Key(row, 0, row.length, cf, 0, cf.length, cq, 0, cq.length, cv, 0, newColVis.getLength(), timestamp);
+ newKey.setDeleted(originalKey.isDeleted());
+ return newKey;
+ }
-
++
+ /**
+ * Make a new key with a column family, column qualifier, and column visibility. Copy the rest of the parts of the key (including delete flag) from
+ * {@code originalKey}.
+ */
+ protected Key replaceKeyParts(Key originalKey, Text newColFam, Text newColQual, Text newColVis) {
+ byte[] row = originalKey.getRowData().toArray();
+ byte[] cf = newColFam.getBytes();
+ byte[] cq = newColQual.getBytes();
+ byte[] cv = newColVis.getBytes();
+ long timestamp = originalKey.getTimestamp();
+ Key newKey = new Key(row, 0, row.length, cf, 0, newColFam.getLength(), cq, 0, newColQual.getLength(), cv, 0, newColVis.getLength(), timestamp);
+ newKey.setDeleted(originalKey.isDeleted());
+ return newKey;
+ }
-
++
+ /**
+ * Make a new key with a column qualifier, and column visibility. Copy the rest of the parts of the key (including delete flag) from {@code originalKey}.
+ */
+ protected Key replaceKeyParts(Key originalKey, Text newColQual, Text newColVis) {
+ byte[] row = originalKey.getRowData().toArray();
+ byte[] cf = originalKey.getColumnFamilyData().toArray();
+ byte[] cq = newColQual.getBytes();
+ byte[] cv = newColVis.getBytes();
+ long timestamp = originalKey.getTimestamp();
+ Key newKey = new Key(row, 0, row.length, cf, 0, cf.length, cq, 0, newColQual.getLength(), cv, 0, newColVis.getLength(), timestamp);
+ newKey.setDeleted(originalKey.isDeleted());
+ return newKey;
+ }
-
++
+ /**
+ * Reverses the transformation applied to column families that are fetched at seek time. If this iterator is transforming column families, then this method
+ * should be overridden to reverse the transformation on the supplied collection of column families. This is necessary since the fetch/seek will be performed
+ * in the transformed space, but when passing the column family set on to the source, the column families need to be in the untransformed space.
- *
++ *
+ * @param columnFamilies
+ * the column families that have been fetched at seek time
+ * @return the untransformed column families that would transform info {@code columnFamilies}
+ */
+ protected Collection<ByteSequence> untransformColumnFamilies(Collection<ByteSequence> columnFamilies) {
+ return columnFamilies;
+ }
-
++
+ /**
+ * Indicates the prefix of keys that will be transformed by this iterator. In other words, this is the part of the key that will <i>not</i> be transformed by
+ * this iterator. For example, if this method returns ROW_COLFAM, then {@link #transformKeys()} may be changing the column qualifier, column visibility, or
+ * timestamp, but it won't be changing the row or column family.
- *
++ *
+ * @return the part of the key this iterator is not transforming
+ */
+ abstract protected PartialKey getKeyPrefix();
-
++
+ public static interface KVBuffer {
+ void append(Key key, Value val);
+ }
-
++
+ /**
+ * Transforms {@code input}. This method must not change the row part of the key, and must only change the parts of the key after the return value of
+ * {@link #getKeyPrefix()}. Implementors must also remember to copy the delete flag from {@code originalKey} onto the new key. Or, implementors should use one
+ * of the helper methods to produce the new key. See any of the replaceKeyParts methods.
- *
++ *
+ * @param input
+ * An iterator over a group of keys with the same prefix. This iterator provides an efficient view, bounded by the prefix, of the underlying iterator
+ * and can not be seeked.
+ * @param output
+ * An output buffer that holds transformed key values. All key values added to the buffer must have the same prefix as the input keys.
- * @throws IOException
+ * @see #replaceColumnFamily(Key, Text)
+ * @see #replaceColumnQualifier(Key, Text)
+ * @see #replaceColumnVisibility(Key, Text)
+ * @see #replaceKeyParts(Key, Text, Text)
+ * @see #replaceKeyParts(Key, Text, Text, Text)
+ */
+ abstract protected void transformRange(SortedKeyValueIterator<Key,Value> input, KVBuffer output) throws IOException;
-
++
+ /**
- * Configure authoriations used for post transformation filtering.
- *
- * @param config
- * @param auths
++ * Configure authorizations used for post transformation filtering.
++ *
+ */
+ public static void setAuthorizations(IteratorSetting config, Authorizations auths) {
+ config.addOption(AUTH_OPT, auths.serialize());
+ }
-
++
+ /**
+ * Configure the maximum amount of memory that can be used for transformation. If this memory is exceeded an exception will be thrown.
- *
- * @param config
++ *
+ * @param maxBufferSize
+ * size in bytes
+ */
+ public static void setMaxBufferSize(IteratorSetting config, long maxBufferSize) {
+ config.addOption(MAX_BUFFER_SIZE_OPT, maxBufferSize + "");
+ }
-
++
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/user/VersioningIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/VersioningIterator.java
index 7804aa4,0000000..2fc3a27
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/VersioningIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/VersioningIterator.java
@@@ -1,172 -1,0 +1,169 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators.user;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.core.iterators.OptionDescriber;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.WrappingIterator;
+
+public class VersioningIterator extends WrappingIterator implements OptionDescriber {
+ private final int maxCount = 10;
+
+ private Key currentKey = new Key();
+ private int numVersions;
+ protected int maxVersions;
+
+ private Range range;
+ private Collection<ByteSequence> columnFamilies;
+ private boolean inclusive;
+
+ @Override
+ public VersioningIterator deepCopy(IteratorEnvironment env) {
+ VersioningIterator copy = new VersioningIterator();
+ copy.setSource(getSource().deepCopy(env));
+ copy.maxVersions = maxVersions;
+ return copy;
+ }
+
+ @Override
+ public void next() throws IOException {
+ if (numVersions >= maxVersions) {
+ skipRowColumn();
+ resetVersionCount();
+ return;
+ }
+
+ super.next();
+ if (getSource().hasTop()) {
+ if (getSource().getTopKey().equals(currentKey, PartialKey.ROW_COLFAM_COLQUAL_COLVIS)) {
+ numVersions++;
+ } else {
+ resetVersionCount();
+ }
+ }
+ }
+
+ @Override
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
+ // do not want to seek to the middle of a row
+ Range seekRange = IteratorUtil.maximizeStartKeyTimeStamp(range);
+ this.range = seekRange;
+ this.columnFamilies = columnFamilies;
+ this.inclusive = inclusive;
+
+ super.seek(seekRange, columnFamilies, inclusive);
+ resetVersionCount();
+
+ if (range.getStartKey() != null)
+ while (hasTop() && range.beforeStartKey(getTopKey()))
+ next();
+ }
+
+ private void resetVersionCount() {
+ if (super.hasTop())
+ currentKey.set(getSource().getTopKey());
+ numVersions = 1;
+ }
+
+ private void skipRowColumn() throws IOException {
+ Key keyToSkip = currentKey;
+ super.next();
+
+ int count = 0;
+ while (getSource().hasTop() && getSource().getTopKey().equals(keyToSkip, PartialKey.ROW_COLFAM_COLQUAL_COLVIS)) {
+ if (count < maxCount) {
+ // it is quicker to call next if we are close, but we never know if we are close
+ // so give next a try a few times
+ getSource().next();
+ count++;
+ } else {
+ reseek(keyToSkip.followingKey(PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
+ count = 0;
+ }
+ }
+ }
+
+ protected void reseek(Key key) throws IOException {
+ if (key == null)
+ return;
+ if (range.afterEndKey(key)) {
+ range = new Range(range.getEndKey(), true, range.getEndKey(), range.isEndKeyInclusive());
+ getSource().seek(range, columnFamilies, inclusive);
+ } else {
+ range = new Range(key, true, range.getEndKey(), range.isEndKeyInclusive());
+ getSource().seek(range, columnFamilies, inclusive);
+ }
+ }
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ super.init(source, options, env);
+ this.numVersions = 0;
+
+ String maxVerString = options.get("maxVersions");
+ if (maxVerString != null)
+ this.maxVersions = Integer.parseInt(maxVerString);
+ else
+ this.maxVersions = 1;
+
+ if (maxVersions < 1)
+ throw new IllegalArgumentException("maxVersions for versioning iterator must be >= 1");
+ }
+
+ @Override
+ public IteratorOptions describeOptions() {
+ return new IteratorOptions("vers", "The VersioningIterator keeps a fixed number of versions for each key", Collections.singletonMap("maxVersions",
+ "number of versions to keep for a particular key (with differing timestamps)"), null);
+ }
+
+ private static final String MAXVERSIONS_OPT = "maxVersions";
+
+ @Override
+ public boolean validateOptions(Map<String,String> options) {
+ int i;
+ try {
+ i = Integer.parseInt(options.get(MAXVERSIONS_OPT));
+ } catch (Exception e) {
+ throw new IllegalArgumentException("bad integer " + MAXVERSIONS_OPT + ":" + options.get(MAXVERSIONS_OPT));
+ }
+ if (i < 1)
+ throw new IllegalArgumentException(MAXVERSIONS_OPT + " for versioning iterator must be >= 1");
+ return true;
+ }
+
+ /**
+ * Encode the maximum number of versions to return onto the ScanIterator
- *
- * @param cfg
- * @param maxVersions
+ */
+ public static void setMaxVersions(IteratorSetting cfg, int maxVersions) {
+ if (maxVersions < 1)
+ throw new IllegalArgumentException(MAXVERSIONS_OPT + " for versioning iterator must be >= 1");
+ cfg.addOption(MAXVERSIONS_OPT, Integer.toString(maxVersions));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/security/SecurityUtil.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/SecurityUtil.java
index 672e784,0000000..65cb7ed
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/security/SecurityUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/SecurityUtil.java
@@@ -1,89 -1,0 +1,88 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.security;
+
+import java.io.IOException;
+import java.net.InetAddress;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.log4j.Logger;
+
+/**
+ *
+ */
+public class SecurityUtil {
+ private static final Logger log = Logger.getLogger(SecurityUtil.class);
+ private static final String ACCUMULO_HOME = "ACCUMULO_HOME", ACCUMULO_CONF_DIR = "ACCUMULO_CONF_DIR";
+ public static boolean usingKerberos = false;
+
+ /**
+ * This method is for logging a server in kerberos. If this is used in client code, it will fail unless run as the accumulo keytab's owner. Instead, use
+ * {@link #login(String, String)}
+ */
+ public static void serverLogin() {
+ @SuppressWarnings("deprecation")
+ AccumuloConfiguration acuConf = AccumuloConfiguration.getSiteConfiguration();
+ String keyTab = acuConf.get(Property.GENERAL_KERBEROS_KEYTAB);
+ if (keyTab == null || keyTab.length() == 0)
+ return;
+
+ usingKerberos = true;
+ if (keyTab.contains("$" + ACCUMULO_HOME) && System.getenv(ACCUMULO_HOME) != null)
+ keyTab = keyTab.replace("$" + ACCUMULO_HOME, System.getenv(ACCUMULO_HOME));
+
+ if (keyTab.contains("$" + ACCUMULO_CONF_DIR) && System.getenv(ACCUMULO_CONF_DIR) != null)
+ keyTab = keyTab.replace("$" + ACCUMULO_CONF_DIR, System.getenv(ACCUMULO_CONF_DIR));
+
+ String principalConfig = acuConf.get(Property.GENERAL_KERBEROS_PRINCIPAL);
+ if (principalConfig == null || principalConfig.length() == 0)
+ return;
+
+ if (login(principalConfig, keyTab)) {
+ try {
+ // This spawns a thread to periodically renew the logged in (accumulo) user
+ UserGroupInformation.getLoginUser();
+ } catch (IOException io) {
+ log.error("Error starting up renewal thread. This shouldn't be happenining.", io);
+ }
+ }
+ }
+
+ /**
+ * This will log in the given user in kerberos.
+ *
+ * @param principalConfig
+ * This is the principals name in the format NAME/HOST@REALM. {@link org.apache.hadoop.security.SecurityUtil#HOSTNAME_PATTERN} will automatically be
+ * replaced by the systems host name.
- * @param keyTabPath
+ * @return true if login succeeded, otherwise false
+ */
+ public static boolean login(String principalConfig, String keyTabPath) {
+ try {
+ String principalName = org.apache.hadoop.security.SecurityUtil.getServerPrincipal(principalConfig, InetAddress.getLocalHost().getCanonicalHostName());
+ if (keyTabPath != null && principalName != null && keyTabPath.length() != 0 && principalName.length() != 0) {
+ UserGroupInformation.loginUserFromKeytab(principalName, keyTabPath);
+ log.info("Succesfully logged in as user " + principalConfig);
+ return true;
+ }
+ } catch (IOException io) {
+ log.error("Error logging in user " + principalConfig + " using keytab at " + keyTabPath, io);
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModule.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModule.java
index fca7d22,0000000..40d3ab7
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModule.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModule.java
@@@ -1,110 -1,0 +1,108 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.security.crypto;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Map;
+
+/**
+ * Classes that obey this interface may be used to provide encrypting and decrypting streams to the rest of Accumulo. Classes that obey this interface may be
+ * configured as the crypto module by setting the property crypto.module.class in the accumulo-site.xml file.
+ *
+ * Note that this first iteration of this API is considered deprecated because we anticipate it changing in non-backwards compatible ways as we explore the
+ * requirements for encryption in Accumulo. So, your mileage is gonna vary a lot as we go forward.
+ *
+ */
+@Deprecated
+public interface CryptoModule {
+
+ public enum CryptoInitProperty {
+ ALGORITHM_NAME("algorithm.name"), CIPHER_SUITE("cipher.suite"), INITIALIZATION_VECTOR("initialization.vector"), PLAINTEXT_SESSION_KEY(
+ "plaintext.session.key");
+
+ private CryptoInitProperty(String name) {
+ key = name;
+ }
+
+ private String key;
+
+ public String getKey() {
+ return key;
+ }
+ }
+
+ /**
+ * Wraps an OutputStream in an encrypting OutputStream. The given map contains the settings for the cryptographic algorithm to use. <b>Callers of this method
+ * should expect that the given OutputStream will be written to before cryptographic writes occur.</b> These writes contain the cryptographic information used
+ * to encrypt the following bytes (these data include the initialization vector, encrypted session key, and so on). If writing arbitrarily to the underlying
+ * stream is not desirable, users should call the other flavor of getEncryptingOutputStream which accepts these data as parameters.
+ *
+ * @param out
+ * the OutputStream to wrap
+ * @param cryptoOpts
+ * the cryptographic parameters to use; specific string names to look for will depend on the various implementations
+ * @return an OutputStream that wraps the given parameter
- * @throws IOException
+ */
+ public OutputStream getEncryptingOutputStream(OutputStream out, Map<String,String> cryptoOpts) throws IOException;
+
+ /**
+ * Wraps an InputStream and returns a decrypting input stream. The given map contains the settings for the intended cryptographic operations, but implementors
+ * should take care to ensure that the crypto from the given input stream matches their expectations about what they will use to decrypt it, as the parameters
+ * may have changed. Also, care should be taken around transitioning between non-encrypting and encrypting streams; implementors should handle the case where
+ * the given input stream is <b>not</b> encrypted at all.
+ *
+ * It is expected that this version of getDecryptingInputStream is called in conjunction with the getEncryptingOutputStream from above. It should expect its
+ * input streams to contain the data written by getEncryptingOutputStream.
+ *
+ * @param in
+ * the InputStream to wrap
+ * @param cryptoOpts
+ * the cryptographic parameters to use; specific string names to look for will depend on the various implementations
+ * @return an InputStream that wraps the given parameter
- * @throws IOException
+ */
+ public InputStream getDecryptingInputStream(InputStream in, Map<String,String> cryptoOpts) throws IOException;
+
+ /**
+ * Wraps an OutputStream in an encrypting OutputStream. The given map contains the settings for the cryptographic algorithm to use. The cryptoInitParams map
+ * contains all the cryptographic details to construct a key (or keys), initialization vectors, etc. and use them to properly initialize the stream for
+ * writing. These initialization parameters must be persisted elsewhere, along with the cryptographic configuration (algorithm, mode, etc.), so that they may
+ * be read in at the time of reading the encrypted content.
+ *
+ * @param out
+ * the OutputStream to wrap
+ * @param conf
+ * the cryptographic algorithm configuration
+ * @param cryptoInitParams
+ * the initialization parameters for the algorithm, usually including initialization vector and session key
+ * @return a wrapped output stream
+ */
+ public OutputStream getEncryptingOutputStream(OutputStream out, Map<String,String> conf, Map<CryptoModule.CryptoInitProperty,Object> cryptoInitParams);
+
+ /**
+ * Wraps an InputStream and returns a decrypting input stream. The given map contains the settings for the intended cryptographic operations, but implementors
+ * should take care to ensure that the crypto from the given input stream matches their expectations about what they will use to decrypt it, as the parameters
+ * may have changed. Also, care should be taken around transitioning between non-encrypting and encrypting streams; implementors should handle the case where
+ * the given input stream is <b>not</b> encrypted at all.
+ *
+ * The cryptoInitParams contains all necessary information to properly initialize the given cipher, usually including things like initialization vector and
+ * secret key.
+ */
+ public InputStream getDecryptingInputStream(InputStream in, Map<String,String> cryptoOpts, Map<CryptoModule.CryptoInitProperty,Object> cryptoInitParams)
+ throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleFactory.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleFactory.java
index 2f03e02,0000000..956c961
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleFactory.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleFactory.java
@@@ -1,254 -1,0 +1,253 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.security.crypto;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Map;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.log4j.Logger;
+
+/**
+ * This factory module exists to assist other classes in loading crypto modules.
+ *
+ * @deprecated This feature is experimental and may go away in future versions.
+ */
+@Deprecated
+public class CryptoModuleFactory {
+
+ private static Logger log = Logger.getLogger(CryptoModuleFactory.class);
+
+ /**
+ * This method returns a crypto module based on settings in the given configuration parameter.
+ *
- * @param conf
+ * @return a class implementing the CryptoModule interface. It will *never* return null; rather, it will return a class which obeys the interface but makes no
+ * changes to the underlying data.
+ */
+ public static CryptoModule getCryptoModule(AccumuloConfiguration conf) {
+ String cryptoModuleClassname = conf.get(Property.CRYPTO_MODULE_CLASS);
+ return getCryptoModule(cryptoModuleClassname);
+ }
+
+ @SuppressWarnings({"rawtypes"})
+ public static CryptoModule getCryptoModule(String cryptoModuleClassname) {
+ log.debug(String.format("About to instantiate crypto module %s", cryptoModuleClassname));
+
+ if (cryptoModuleClassname.equals("NullCryptoModule")) {
+ return new NullCryptoModule();
+ }
+
+ CryptoModule cryptoModule = null;
+ Class cryptoModuleClazz = null;
+ try {
+ cryptoModuleClazz = AccumuloVFSClassLoader.loadClass(cryptoModuleClassname);
+ } catch (ClassNotFoundException e1) {
+ log.warn(String.format("Could not find configured crypto module \"%s\". NO ENCRYPTION WILL BE USED.", cryptoModuleClassname));
+ return new NullCryptoModule();
+ }
+
+ // Check if the given class implements the CryptoModule interface
+ Class[] interfaces = cryptoModuleClazz.getInterfaces();
+ boolean implementsCryptoModule = false;
+
+ for (Class clazz : interfaces) {
+ if (clazz.equals(CryptoModule.class)) {
+ implementsCryptoModule = true;
+ break;
+ }
+ }
+
+ if (!implementsCryptoModule) {
+ log.warn("Configured Accumulo crypto module \"%s\" does not implement the CryptoModule interface. NO ENCRYPTION WILL BE USED.");
+ return new NullCryptoModule();
+ } else {
+ try {
+ cryptoModule = (CryptoModule) cryptoModuleClazz.newInstance();
+
+ log.debug("Successfully instantiated crypto module");
+
+ } catch (InstantiationException e) {
+ log.warn(String.format("Got instantiation exception %s when instantiating crypto module \"%s\". NO ENCRYPTION WILL BE USED.", e.getCause().getClass()
+ .getCanonicalName(), cryptoModuleClassname));
+ log.warn(e.getCause());
+ return new NullCryptoModule();
+ } catch (IllegalAccessException e) {
+ log.warn(String.format("Got illegal access exception when trying to instantiate crypto module \"%s\". NO ENCRYPTION WILL BE USED.",
+ cryptoModuleClassname));
+ log.warn(e);
+ return new NullCryptoModule();
+ }
+ }
+ return cryptoModule;
+ }
+
+ public static SecretKeyEncryptionStrategy getSecretKeyEncryptionStrategy(AccumuloConfiguration conf) {
+ String className = conf.get(Property.CRYPTO_SECRET_KEY_ENCRYPTION_STRATEGY_CLASS);
+ return getSecretKeyEncryptionStrategy(className);
+ }
+
+ @SuppressWarnings("rawtypes")
+ public static SecretKeyEncryptionStrategy getSecretKeyEncryptionStrategy(String className) {
+ if (className == null || className.equals("NullSecretKeyEncryptionStrategy")) {
+ return new NullSecretKeyEncryptionStrategy();
+ }
+
+ SecretKeyEncryptionStrategy strategy = null;
+ Class keyEncryptionStrategyClazz = null;
+ try {
+ keyEncryptionStrategyClazz = AccumuloVFSClassLoader.loadClass(className);
+ } catch (ClassNotFoundException e1) {
+ log.warn(String.format("Could not find configured secret key encryption strategy \"%s\". NO ENCRYPTION WILL BE USED.", className));
+ return new NullSecretKeyEncryptionStrategy();
+ }
+
+ // Check if the given class implements the CryptoModule interface
+ Class[] interfaces = keyEncryptionStrategyClazz.getInterfaces();
+ boolean implementsSecretKeyStrategy = false;
+
+ for (Class clazz : interfaces) {
+ if (clazz.equals(SecretKeyEncryptionStrategy.class)) {
+ implementsSecretKeyStrategy = true;
+ break;
+ }
+ }
+
+ if (!implementsSecretKeyStrategy) {
+ log.warn("Configured Accumulo secret key encryption strategy \"%s\" does not implement the SecretKeyEncryptionStrategy interface. NO ENCRYPTION WILL BE USED.");
+ return new NullSecretKeyEncryptionStrategy();
+ } else {
+ try {
+ strategy = (SecretKeyEncryptionStrategy) keyEncryptionStrategyClazz.newInstance();
+
+ log.debug("Successfully instantiated secret key encryption strategy");
+
+ } catch (InstantiationException e) {
+ log.warn(String.format("Got instantiation exception %s when instantiating secret key encryption strategy \"%s\". NO ENCRYPTION WILL BE USED.", e
+ .getCause().getClass().getCanonicalName(), className));
+ log.warn(e.getCause());
+ return new NullSecretKeyEncryptionStrategy();
+ } catch (IllegalAccessException e) {
+ log.warn(String.format("Got illegal access exception when trying to instantiate secret key encryption strategy \"%s\". NO ENCRYPTION WILL BE USED.",
+ className));
+ log.warn(e);
+ return new NullSecretKeyEncryptionStrategy();
+ }
+ }
+
+ return strategy;
+ }
+
+ private static class NullSecretKeyEncryptionStrategy implements SecretKeyEncryptionStrategy {
+
+ @Override
+ public SecretKeyEncryptionStrategyContext encryptSecretKey(SecretKeyEncryptionStrategyContext context) {
+ context.setEncryptedSecretKey(context.getPlaintextSecretKey());
+ context.setOpaqueKeyEncryptionKeyID("");
+
+ return context;
+ }
+
+ @Override
+ public SecretKeyEncryptionStrategyContext decryptSecretKey(SecretKeyEncryptionStrategyContext context) {
+ context.setPlaintextSecretKey(context.getEncryptedSecretKey());
+
+ return context;
+ }
+
+ @Override
+ public SecretKeyEncryptionStrategyContext getNewContext() {
+ return new SecretKeyEncryptionStrategyContext() {
+
+ @Override
+ public byte[] getPlaintextSecretKey() {
+ return plaintextSecretKey;
+ }
+
+ @Override
+ public void setPlaintextSecretKey(byte[] plaintextSecretKey) {
+ this.plaintextSecretKey = plaintextSecretKey;
+ }
+
+ @Override
+ public byte[] getEncryptedSecretKey() {
+ return encryptedSecretKey;
+ }
+
+ @Override
+ public void setEncryptedSecretKey(byte[] encryptedSecretKey) {
+ this.encryptedSecretKey = encryptedSecretKey;
+ }
+
+ @Override
+ public String getOpaqueKeyEncryptionKeyID() {
+ return opaqueKeyEncryptionKeyID;
+ }
+
+ @Override
+ public void setOpaqueKeyEncryptionKeyID(String opaqueKeyEncryptionKeyID) {
+ this.opaqueKeyEncryptionKeyID = opaqueKeyEncryptionKeyID;
+ }
+
+ @Override
+ public Map<String,String> getContext() {
+ return context;
+ }
+
+ @Override
+ public void setContext(Map<String,String> context) {
+ this.context = context;
+ }
+
+ private byte[] plaintextSecretKey;
+ private byte[] encryptedSecretKey;
+ private String opaqueKeyEncryptionKeyID;
+ private Map<String,String> context;
+ };
+ }
+
+ }
+
+ private static class NullCryptoModule implements CryptoModule {
+
+ @Override
+ public OutputStream getEncryptingOutputStream(OutputStream out, Map<String,String> cryptoOpts) throws IOException {
+ return out;
+ }
+
+ @Override
+ public InputStream getDecryptingInputStream(InputStream in, Map<String,String> cryptoOpts) throws IOException {
+ return in;
+ }
+
+ @Override
+ public OutputStream getEncryptingOutputStream(OutputStream out, Map<String,String> conf, Map<CryptoInitProperty,Object> cryptoInitParams) {
+ return out;
+ }
+
+ @Override
+ public InputStream getDecryptingInputStream(InputStream in, Map<String,String> cryptoOpts, Map<CryptoInitProperty,Object> cryptoInitParams)
+ throws IOException {
+ return in;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/test/java/org/apache/accumulo/core/client/impl/ScannerOptionsTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/impl/ScannerOptionsTest.java
index 07bd518,0000000..463822a
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/impl/ScannerOptionsTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/impl/ScannerOptionsTest.java
@@@ -1,59 -1,0 +1,57 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.impl;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.iterators.DebugIterator;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.junit.Test;
+
+/**
+ * Test that scanner options are set/unset correctly
+ */
+public class ScannerOptionsTest {
+
+ /**
+ * Test that you properly add and remove iterators from a scanner
- *
- * @throws Throwable
+ */
+ @Test
+ public void testAddRemoveIterator() throws Throwable {
+ ScannerOptions options = new ScannerOptions();
+ options.addScanIterator(new IteratorSetting(1, "NAME", WholeRowIterator.class));
+ assertEquals(1, options.serverSideIteratorList.size());
+ options.removeScanIterator("NAME");
+ assertEquals(0, options.serverSideIteratorList.size());
+ }
+
+ @Test
+ public void testIteratorConflict() {
+ ScannerOptions options = new ScannerOptions();
+ options.addScanIterator(new IteratorSetting(1, "NAME", DebugIterator.class));
+ try {
+ options.addScanIterator(new IteratorSetting(2, "NAME", DebugIterator.class));
+ fail();
+ } catch (IllegalArgumentException e) {}
+ try {
+ options.addScanIterator(new IteratorSetting(1, "NAME2", DebugIterator.class));
+ fail();
+ } catch (IllegalArgumentException e) {}
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
index 4f527e1,0000000..3ec9bb1
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
@@@ -1,284 -1,0 +1,280 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapred;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.lib.NullOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.Test;
+
+public class AccumuloInputFormatTest {
+
+ private static final String PREFIX = AccumuloInputFormatTest.class.getSimpleName();
+ private static final String INSTANCE_NAME = PREFIX + "_mapred_instance";
+ private static final String TEST_TABLE_1 = PREFIX + "_mapred_table_1";
+
+ /**
+ * Check that the iterator configuration is getting stored in the Job conf correctly.
- *
- * @throws IOException
+ */
+ @Test
+ public void testSetIterator() throws IOException {
+ JobConf job = new JobConf();
+
+ IteratorSetting is = new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator");
+ AccumuloInputFormat.addIterator(job, is);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ is.write(new DataOutputStream(baos));
+ String iterators = job.get("AccumuloInputFormat.ScanOpts.Iterators");
+ assertEquals(new String(Base64.encodeBase64(baos.toByteArray())), iterators);
+ }
+
+ @Test
+ public void testAddIterator() throws IOException {
+ JobConf job = new JobConf();
+
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", WholeRowIterator.class));
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
+ IteratorSetting iter = new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator");
+ iter.addOption("v1", "1");
+ iter.addOption("junk", "\0omg:!\\xyzzy");
+ AccumuloInputFormat.addIterator(job, iter);
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+
+ // Check the list size
+ assertTrue(list.size() == 3);
+
+ // Walk the list and make sure our settings are correct
+ IteratorSetting setting = list.get(0);
+ assertEquals(1, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.user.WholeRowIterator", setting.getIteratorClass());
+ assertEquals("WholeRow", setting.getName());
+ assertEquals(0, setting.getOptions().size());
+
+ setting = list.get(1);
+ assertEquals(2, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
+ assertEquals("Versions", setting.getName());
+ assertEquals(0, setting.getOptions().size());
+
+ setting = list.get(2);
+ assertEquals(3, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
+ assertEquals("Count", setting.getName());
+ assertEquals(2, setting.getOptions().size());
+ assertEquals("1", setting.getOptions().get("v1"));
+ assertEquals("\0omg:!\\xyzzy", setting.getOptions().get("junk"));
+ }
+
+ /**
+ * Test adding iterator options where the keys and values contain both the FIELD_SEPARATOR character (':') and ITERATOR_SEPARATOR (',') characters. There
+ * should be no exceptions thrown when trying to parse these types of option entries.
+ *
+ * This test makes sure that the expected raw values, as appears in the Job, are equal to what's expected.
+ */
+ @Test
+ public void testIteratorOptionEncoding() throws Throwable {
+ String key = "colon:delimited:key";
+ String value = "comma,delimited,value";
+ IteratorSetting someSetting = new IteratorSetting(1, "iterator", "Iterator.class");
+ someSetting.addOption(key, value);
+ JobConf job = new JobConf();
+ AccumuloInputFormat.addIterator(job, someSetting);
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+ assertEquals(1, list.size());
+ assertEquals(1, list.get(0).getOptions().size());
+ assertEquals(list.get(0).getOptions().get(key), value);
+
+ someSetting.addOption(key + "2", value);
+ someSetting.setPriority(2);
+ someSetting.setName("it2");
+ AccumuloInputFormat.addIterator(job, someSetting);
+ list = AccumuloInputFormat.getIterators(job);
+ assertEquals(2, list.size());
+ assertEquals(1, list.get(0).getOptions().size());
+ assertEquals(list.get(0).getOptions().get(key), value);
+ assertEquals(2, list.get(1).getOptions().size());
+ assertEquals(list.get(1).getOptions().get(key), value);
+ assertEquals(list.get(1).getOptions().get(key + "2"), value);
+ }
+
+ /**
+ * Test getting iterator settings for multiple iterators set
- *
- * @throws IOException
+ */
+ @Test
+ public void testGetIteratorSettings() throws IOException {
+ JobConf job = new JobConf();
+
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator"));
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator"));
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+
+ // Check the list size
+ assertTrue(list.size() == 3);
+
+ // Walk the list and make sure our settings are correct
+ IteratorSetting setting = list.get(0);
+ assertEquals(1, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.WholeRowIterator", setting.getIteratorClass());
+ assertEquals("WholeRow", setting.getName());
+
+ setting = list.get(1);
+ assertEquals(2, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
+ assertEquals("Versions", setting.getName());
+
+ setting = list.get(2);
+ assertEquals(3, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
+ assertEquals("Count", setting.getName());
+
+ }
+
+ @Test
+ public void testSetRegex() throws IOException {
+ JobConf job = new JobConf();
+
+ String regex = ">\"*%<>\'\\";
+
+ IteratorSetting is = new IteratorSetting(50, regex, RegExFilter.class);
+ RegExFilter.setRegexs(is, regex, null, null, null, false);
+ AccumuloInputFormat.addIterator(job, is);
+
+ assertTrue(regex.equals(AccumuloInputFormat.getIterators(job).get(0).getName()));
+ }
+
+ private static AssertionError e1 = null;
+ private static AssertionError e2 = null;
+
+ private static class MRTester extends Configured implements Tool {
+ private static class TestMapper implements Mapper<Key,Value,Key,Value> {
+ Key key = null;
+ int count = 0;
+
+ @Override
+ public void map(Key k, Value v, OutputCollector<Key,Value> output, Reporter reporter) throws IOException {
+ try {
+ if (key != null)
+ assertEquals(key.getRow().toString(), new String(v.get()));
+ assertEquals(k.getRow(), new Text(String.format("%09x", count + 1)));
+ assertEquals(new String(v.get()), String.format("%09x", count));
+ } catch (AssertionError e) {
+ e1 = e;
+ }
+ key = new Key(k);
+ count++;
+ }
+
+ @Override
+ public void configure(JobConf job) {}
+
+ @Override
+ public void close() throws IOException {
+ try {
+ assertEquals(100, count);
+ } catch (AssertionError e) {
+ e2 = e;
+ }
+ }
+
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+
+ if (args.length != 3) {
+ throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <user> <pass> <table>");
+ }
+
+ String user = args[0];
+ String pass = args[1];
+ String table = args[2];
+
+ JobConf job = new JobConf(getConf());
+ job.setJarByClass(this.getClass());
+
+ job.setInputFormat(AccumuloInputFormat.class);
+
+ AccumuloInputFormat.setConnectorInfo(job, user, new PasswordToken(pass));
+ AccumuloInputFormat.setInputTableName(job, table);
+ AccumuloInputFormat.setMockInstance(job, INSTANCE_NAME);
+
+ job.setMapperClass(TestMapper.class);
+ job.setMapOutputKeyClass(Key.class);
+ job.setMapOutputValueClass(Value.class);
+ job.setOutputFormat(NullOutputFormat.class);
+
+ job.setNumReduceTasks(0);
+
+ return JobClient.runJob(job).isSuccessful() ? 0 : 1;
+ }
+
+ public static void main(String[] args) throws Exception {
+ assertEquals(0, ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args));
+ }
+ }
+
+ @Test
+ public void testMap() throws Exception {
+ MockInstance mockInstance = new MockInstance(INSTANCE_NAME);
+ Connector c = mockInstance.getConnector("root", new PasswordToken(""));
+ c.tableOperations().create(TEST_TABLE_1);
+ BatchWriter bw = c.createBatchWriter(TEST_TABLE_1, new BatchWriterConfig());
+ for (int i = 0; i < 100; i++) {
+ Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
+ m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
+ bw.addMutation(m);
+ }
+ bw.close();
+
+ MRTester.main(new String[] {"root", "", TEST_TABLE_1});
+ assertNull(e1);
+ assertNull(e2);
+ }
+}
[39/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
index 36bd86a,0000000..8873938
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
+++ b/server/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
@@@ -1,157 -1,0 +1,156 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security.handler;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+public class ZKAuthorizor implements Authorizor {
+ private static final Logger log = Logger.getLogger(ZKAuthorizor.class);
+ private static Authorizor zkAuthorizorInstance = null;
+
+ private final String ZKUserAuths = "/Authorizations";
+
+ private String ZKUserPath;
+ private final ZooCache zooCache;
+
+ public static synchronized Authorizor getInstance() {
+ if (zkAuthorizorInstance == null)
+ zkAuthorizorInstance = new ZKAuthorizor();
+ return zkAuthorizorInstance;
+ }
+
+ public ZKAuthorizor() {
+ zooCache = new ZooCache();
+ }
+
++ @Override
+ public void initialize(String instanceId, boolean initialize) {
+ ZKUserPath = ZKSecurityTool.getInstancePath(instanceId) + "/users";
+ }
+
++ @Override
+ public Authorizations getCachedUserAuthorizations(String user) {
+ byte[] authsBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserAuths);
+ if (authsBytes != null)
+ return ZKSecurityTool.convertAuthorizations(authsBytes);
+ return Constants.NO_AUTHS;
+ }
+
+ @Override
+ public boolean validSecurityHandlers(Authenticator auth, PermissionHandler pm) {
+ return true;
+ }
+
+ @Override
+ public void initializeSecurity(TCredentials itw, String rootuser) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+
+ // create the root user with all system privileges, no table privileges, and no record-level authorizations
+ Set<SystemPermission> rootPerms = new TreeSet<SystemPermission>();
+ for (SystemPermission p : SystemPermission.values())
+ rootPerms.add(p);
+ Map<String,Set<TablePermission>> tablePerms = new HashMap<String,Set<TablePermission>>();
+ // Allow the root user to flush the !METADATA table
+ tablePerms.put(Constants.METADATA_TABLE_ID, Collections.singleton(TablePermission.ALTER_TABLE));
+
+ try {
+ // prep parent node of users with root username
+ if (!zoo.exists(ZKUserPath))
+ zoo.putPersistentData(ZKUserPath, rootuser.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
+
+ initUser(rootuser);
+ zoo.putPersistentData(ZKUserPath + "/" + rootuser + ZKUserAuths, ZKSecurityTool.convertAuthorizations(Constants.NO_AUTHS), NodeExistsPolicy.FAIL);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
- /**
- * @param user
- * @throws AccumuloSecurityException
- */
++ @Override
+ public void initUser(String user) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ try {
+ zoo.putPersistentData(ZKUserPath + "/" + user, new byte[0], NodeExistsPolicy.SKIP);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void dropUser(String user) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserAuths, NodeMissingPolicy.SKIP);
+ zooCache.clear(ZKUserPath + "/" + user);
+ }
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ if (e.code().equals(KeeperException.Code.NONODE))
+ throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+
+ }
+ }
+
+ @Override
+ public void changeAuthorizations(String user, Authorizations authorizations) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserAuths, ZKSecurityTool.convertAuthorizations(authorizations),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
index 4a05658,0000000..d802eb9
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
+++ b/server/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
@@@ -1,365 -1,0 +1,363 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security.handler;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.Code;
+
+/**
+ *
+ */
+public class ZKPermHandler implements PermissionHandler {
+ private static final Logger log = Logger.getLogger(ZKAuthorizor.class);
+ private static PermissionHandler zkPermHandlerInstance = null;
+
+ private String ZKUserPath;
+ private String ZKTablePath;
+ private final ZooCache zooCache;
+ private static final String ZKUserSysPerms = "/System";
+ private static final String ZKUserTablePerms = "/Tables";
+
+ public static synchronized PermissionHandler getInstance() {
+ if (zkPermHandlerInstance == null)
+ zkPermHandlerInstance = new ZKPermHandler();
+ return zkPermHandlerInstance;
+ }
+
++ @Override
+ public void initialize(String instanceId, boolean initialize) {
+ ZKUserPath = ZKSecurityTool.getInstancePath(instanceId) + "/users";
+ ZKTablePath = ZKSecurityTool.getInstancePath(instanceId) + "/tables";
+ }
+
+ public ZKPermHandler() {
+ zooCache = new ZooCache();
+ }
+
+ @Override
+ public boolean hasTablePermission(String user, String table, TablePermission permission) throws TableNotFoundException {
+ byte[] serializedPerms;
+ try {
+ String path = ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table;
+ ZooReaderWriter.getRetryingInstance().sync(path);
+ serializedPerms = ZooReaderWriter.getRetryingInstance().getData(path, null);
+ } catch (KeeperException e) {
+ if (e.code() == Code.NONODE) {
+ // maybe the table was just deleted?
+ try {
+ // check for existence:
+ ZooReaderWriter.getRetryingInstance().getData(ZKTablePath + "/" + table, null);
+ // it's there, you don't have permission
+ return false;
+ } catch (InterruptedException ex) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ } catch (KeeperException ex) {
+ // not there, throw an informative exception
+ if (e.code() == Code.NONODE) {
+ throw new TableNotFoundException(null, table, "while checking permissions");
+ }
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ }
+ return false;
+ }
+ log.warn("Unhandled KeeperException, failing closed for table permission check", e);
+ return false;
+ } catch (InterruptedException e) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ }
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertTablePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean hasCachedTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertTablePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public void grantSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ try {
+ byte[] permBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+ Set<SystemPermission> perms;
+ if (permBytes == null) {
+ perms = new TreeSet<SystemPermission>();
+ } else {
+ perms = ZKSecurityTool.convertSystemPermissions(permBytes);
+ }
+
+ if (perms.add(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(perms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void grantTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException {
+ Set<TablePermission> tablePerms;
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ if (serializedPerms != null)
+ tablePerms = ZKSecurityTool.convertTablePermissions(serializedPerms);
+ else
+ tablePerms = new TreeSet<TablePermission>();
+
+ try {
+ if (tablePerms.add(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, ZKSecurityTool.convertTablePermissions(tablePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void revokeSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] sysPermBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+
+ // User had no system permission, nothing to revoke.
+ if (sysPermBytes == null)
+ return;
+
+ Set<SystemPermission> sysPerms = ZKSecurityTool.convertSystemPermissions(sysPermBytes);
+
+ try {
+ if (sysPerms.remove(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(sysPerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void revokeTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+
+ // User had no table permission, nothing to revoke.
+ if (serializedPerms == null)
+ return;
+
+ Set<TablePermission> tablePerms = ZKSecurityTool.convertTablePermissions(serializedPerms);
+ try {
+ if (tablePerms.remove(permission)) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ if (tablePerms.size() == 0)
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, NodeMissingPolicy.SKIP);
+ else
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, ZKSecurityTool.convertTablePermissions(tablePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void cleanTablePermissions(String table) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ for (String user : zooCache.getChildren(ZKUserPath))
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, NodeMissingPolicy.SKIP);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException("unknownUser", SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void initializeSecurity(TCredentials itw, String rootuser) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+
+ // create the root user with all system privileges, no table privileges, and no record-level authorizations
+ Set<SystemPermission> rootPerms = new TreeSet<SystemPermission>();
+ for (SystemPermission p : SystemPermission.values())
+ rootPerms.add(p);
+ Map<String,Set<TablePermission>> tablePerms = new HashMap<String,Set<TablePermission>>();
+ // Allow the root user to flush the !METADATA table
+ tablePerms.put(Constants.METADATA_TABLE_ID, Collections.singleton(TablePermission.ALTER_TABLE));
+
+ try {
+ // prep parent node of users with root username
+ if (!zoo.exists(ZKUserPath))
+ zoo.putPersistentData(ZKUserPath, rootuser.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
+
+ initUser(rootuser);
+ zoo.putPersistentData(ZKUserPath + "/" + rootuser + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(rootPerms), NodeExistsPolicy.FAIL);
+ for (Entry<String,Set<TablePermission>> entry : tablePerms.entrySet())
+ createTablePerm(rootuser, entry.getKey(), entry.getValue());
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
- /**
- * @param user
- * @throws AccumuloSecurityException
- */
++ @Override
+ public void initUser(String user) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ try {
+ zoo.putPersistentData(ZKUserPath + "/" + user, new byte[0], NodeExistsPolicy.SKIP);
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms, new byte[0], NodeExistsPolicy.SKIP);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Sets up a new table configuration for the provided user/table. No checking for existence is done here, it should be done before calling.
+ */
+ private void createTablePerm(String user, String table, Set<TablePermission> perms) throws KeeperException, InterruptedException {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table,
+ ZKSecurityTool.convertTablePermissions(perms), NodeExistsPolicy.FAIL);
+ }
+ }
+
+ @Override
+ public void cleanUser(String user) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserSysPerms, NodeMissingPolicy.SKIP);
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms, NodeMissingPolicy.SKIP);
+ zooCache.clear(ZKUserPath + "/" + user);
+ }
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ if (e.code().equals(KeeperException.Code.NONODE))
+ throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+
+ }
+ }
+
+ @Override
+ public boolean hasSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] perms;
+ try {
+ String path = ZKUserPath + "/" + user + ZKUserSysPerms;
+ ZooReaderWriter.getRetryingInstance().sync(path);
+ perms = ZooReaderWriter.getRetryingInstance().getData(path, null);
+ } catch (KeeperException e) {
+ if (e.code() == Code.NONODE) {
+ return false;
+ }
+ log.warn("Unhandled KeeperException, failing closed for table permission check", e);
+ return false;
+ } catch (InterruptedException e) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ }
+
+ if (perms == null)
+ return false;
+ return ZKSecurityTool.convertSystemPermissions(perms).contains(permission);
+ }
+
+ @Override
+ public boolean hasCachedSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] perms = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+ if (perms == null)
+ return false;
+ return ZKSecurityTool.convertSystemPermissions(perms).contains(permission);
+ }
+
+ @Override
+ public boolean validSecurityHandlers(Authenticator authent, Authorizor author) {
+ return true;
+ }
+
+ @Override
+ public void initTable(String table) throws AccumuloSecurityException {
+ // All proper housekeeping is done on delete and permission granting, no work needs to be done here
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/tabletserver/MemValue.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/tabletserver/MemValue.java
index 735bf20,0000000..f68859a
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/MemValue.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/MemValue.java
@@@ -1,93 -1,0 +1,95 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.tabletserver;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.accumulo.core.data.Value;
+
+/**
+ *
+ */
+public class MemValue extends Value {
+ int kvCount;
+ boolean merged = false;
+
+ /**
+ * @param value
+ * Value
+ * @param kv
+ * kv count
+ */
+ public MemValue(byte[] value, int kv) {
+ super(value);
+ this.kvCount = kv;
+ }
+
+ public MemValue() {
+ super();
+ this.kvCount = Integer.MAX_VALUE;
+ }
+
+ public MemValue(Value value, int kv) {
+ super(value);
+ this.kvCount = kv;
+ }
+
+ // Override
++ @Override
+ public void write(final DataOutput out) throws IOException {
+ if (!merged) {
+ byte[] combinedBytes = new byte[getSize() + 4];
+ System.arraycopy(value, 0, combinedBytes, 4, getSize());
+ combinedBytes[0] = (byte) (kvCount >>> 24);
+ combinedBytes[1] = (byte) (kvCount >>> 16);
+ combinedBytes[2] = (byte) (kvCount >>> 8);
+ combinedBytes[3] = (byte) (kvCount);
+ value = combinedBytes;
+ merged = true;
+ }
+ super.write(out);
+ }
+
++ @Override
+ public void set(final byte[] b) {
+ super.set(b);
+ merged = false;
+ }
+
++ @Override
+ public void copy(byte[] b) {
+ super.copy(b);
+ merged = false;
+ }
+
+ /**
+ * Takes a Value and will take out the embedded kvCount, and then return that value while replacing the Value with the original unembedded version
+ *
- * @param v
+ * @return The kvCount embedded in v.
+ */
+ public static int splitKVCount(Value v) {
+ if (v instanceof MemValue)
+ return ((MemValue) v).kvCount;
+
+ byte[] originalBytes = new byte[v.getSize() - 4];
+ byte[] combined = v.get();
+ System.arraycopy(combined, 4, originalBytes, 0, originalBytes.length);
+ v.set(originalBytes);
+ return (combined[0] << 24) + ((combined[1] & 0xFF) << 16) + ((combined[2] & 0xFF) << 8) + (combined[3] & 0xFF);
+ }
+}
[47/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
index 9c02219,0000000..1f83541
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
@@@ -1,681 -1,0 +1,680 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.mapreduce.lib.util.OutputConfigurator;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.SecurityErrorCode;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * This class allows MapReduce jobs to use Accumulo as the sink for data. This {@link OutputFormat} accepts keys and values of type {@link Text} (for a table
+ * name) and {@link Mutation} from the Map and Reduce functions.
+ *
+ * The user must specify the following via static configurator methods:
+ *
+ * <ul>
+ * <li>{@link AccumuloOutputFormat#setConnectorInfo(Job, String, AuthenticationToken)}
+ * <li>{@link AccumuloOutputFormat#setZooKeeperInstance(Job, String, String)} OR {@link AccumuloOutputFormat#setMockInstance(Job, String)}
+ * </ul>
+ *
+ * Other static methods are optional.
+ */
+public class AccumuloOutputFormat extends OutputFormat<Text,Mutation> {
+
+ private static final Class<?> CLASS = AccumuloOutputFormat.class;
+ protected static final Logger log = Logger.getLogger(CLASS);
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
+ * conversion to a string, and is not intended to be secure.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission if {@link #setCreateTables(Job, boolean)} is set to true)
+ * @param token
+ * the user's password
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public static void setConnectorInfo(Job job, String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ OutputConfigurator.setConnectorInfo(CLASS, job.getConfiguration(), principal, token);
+ }
+
+ /**
+ * Determines if the connector has been configured.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the connector has been configured, false otherwise
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static Boolean isConnectorInfoSet(JobContext context) {
+ return OutputConfigurator.isConnectorInfoSet(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static String getPrincipal(JobContext context) {
+ return OutputConfigurator.getPrincipal(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Gets the serialized token class name from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static String getTokenClass(JobContext context) {
+ return OutputConfigurator.getTokenClass(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Gets the password from the configuration. WARNING: The password is stored in the Configuration and shared with all MapReduce tasks; It is BASE64 encoded to
+ * provide a charset safe conversion to a string, and is not intended to be secure.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the decoded user password
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static byte[] getToken(JobContext context) {
+ return OutputConfigurator.getToken(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Configures a {@link ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @param zooKeepers
+ * a comma-separated list of zookeeper servers
+ * @since 1.5.0
+ */
+ public static void setZooKeeperInstance(Job job, String instanceName, String zooKeepers) {
+ OutputConfigurator.setZooKeeperInstance(CLASS, job.getConfiguration(), instanceName, zooKeepers);
+ }
+
+ /**
+ * Configures a {@link MockInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @since 1.5.0
+ */
+ public static void setMockInstance(Job job, String instanceName) {
+ OutputConfigurator.setMockInstance(CLASS, job.getConfiguration(), instanceName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link Instance} based on the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return an Accumulo instance
+ * @since 1.5.0
+ * @see #setZooKeeperInstance(Job, String, String)
+ * @see #setMockInstance(Job, String)
+ */
+ protected static Instance getInstance(JobContext context) {
+ return OutputConfigurator.getInstance(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param level
+ * the logging level
+ * @since 1.5.0
+ */
+ public static void setLogLevel(Job job, Level level) {
+ OutputConfigurator.setLogLevel(CLASS, job.getConfiguration(), level);
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the log level
+ * @since 1.5.0
+ * @see #setLogLevel(Job, Level)
+ */
+ protected static Level getLogLevel(JobContext context) {
+ return OutputConfigurator.getLogLevel(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Sets the default table name to use if one emits a null in place of a table name for a given mutation. Table names can only be alpha-numeric and
+ * underscores.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.5.0
+ */
+ public static void setDefaultTableName(Job job, String tableName) {
+ OutputConfigurator.setDefaultTableName(CLASS, job.getConfiguration(), tableName);
+ }
+
+ /**
+ * Gets the default table name from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the default table name
+ * @since 1.5.0
+ * @see #setDefaultTableName(Job, String)
+ */
+ protected static String getDefaultTableName(JobContext context) {
+ return OutputConfigurator.getDefaultTableName(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Sets the configuration for for the job's {@link BatchWriter} instances. If not set, a new {@link BatchWriterConfig}, with sensible built-in defaults is
+ * used. Setting the configuration multiple times overwrites any previous configuration.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param bwConfig
+ * the configuration for the {@link BatchWriter}
+ * @since 1.5.0
+ */
+ public static void setBatchWriterOptions(Job job, BatchWriterConfig bwConfig) {
+ OutputConfigurator.setBatchWriterOptions(CLASS, job.getConfiguration(), bwConfig);
+ }
+
+ /**
+ * Gets the {@link BatchWriterConfig} settings.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the configuration object
+ * @since 1.5.0
+ * @see #setBatchWriterOptions(Job, BatchWriterConfig)
+ */
+ protected static BatchWriterConfig getBatchWriterOptions(JobContext context) {
+ return OutputConfigurator.getBatchWriterOptions(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Sets the directive to create new tables, as necessary. Table names can only be alpha-numeric and underscores.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setCreateTables(Job job, boolean enableFeature) {
+ OutputConfigurator.setCreateTables(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether tables are permitted to be created as needed.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the feature is disabled, false otherwise
+ * @since 1.5.0
+ * @see #setCreateTables(Job, boolean)
+ */
+ protected static Boolean canCreateTables(JobContext context) {
+ return OutputConfigurator.canCreateTables(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Sets the directive to use simulation mode for this job. In simulation mode, no output is produced. This is useful for testing.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setSimulationMode(Job job, boolean enableFeature) {
+ OutputConfigurator.setSimulationMode(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether this feature is enabled.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setSimulationMode(Job, boolean)
+ */
+ protected static Boolean getSimulationMode(JobContext context) {
+ return OutputConfigurator.getSimulationMode(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * A base class to be used to create {@link RecordWriter} instances that write to Accumulo.
+ */
+ protected static class AccumuloRecordWriter extends RecordWriter<Text,Mutation> {
+ private MultiTableBatchWriter mtbw = null;
+ private HashMap<Text,BatchWriter> bws = null;
+ private Text defaultTableName = null;
+
+ private boolean simulate = false;
+ private boolean createTables = false;
+
+ private long mutCount = 0;
+ private long valCount = 0;
+
+ private Connector conn;
+
+ protected AccumuloRecordWriter(TaskAttemptContext context) throws AccumuloException, AccumuloSecurityException, IOException {
+ Level l = getLogLevel(context);
+ if (l != null)
+ log.setLevel(getLogLevel(context));
+ this.simulate = getSimulationMode(context);
+ this.createTables = canCreateTables(context);
+
+ if (simulate)
+ log.info("Simulating output only. No writes to tables will occur");
+
+ this.bws = new HashMap<Text,BatchWriter>();
+
+ String tname = getDefaultTableName(context);
+ this.defaultTableName = (tname == null) ? null : new Text(tname);
+
+ if (!simulate) {
+ this.conn = getInstance(context).getConnector(getPrincipal(context), CredentialHelper.extractToken(getTokenClass(context), getToken(context)));
+ mtbw = conn.createMultiTableBatchWriter(getBatchWriterOptions(context));
+ }
+ }
+
+ /**
+ * Push a mutation into a table. If table is null, the defaultTable will be used. If canCreateTable is set, the table will be created if it does not exist.
+ * The table name must only contain alphanumerics and underscore.
+ */
+ @Override
+ public void write(Text table, Mutation mutation) throws IOException {
+ if (table == null || table.toString().isEmpty())
+ table = this.defaultTableName;
+
+ if (!simulate && table == null)
+ throw new IOException("No table or default table specified. Try simulation mode next time");
+
+ ++mutCount;
+ valCount += mutation.size();
+ printMutation(table, mutation);
+
+ if (simulate)
+ return;
+
+ if (!bws.containsKey(table))
+ try {
+ addTable(table);
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new IOException(e);
+ }
+
+ try {
+ bws.get(table).addMutation(mutation);
+ } catch (MutationsRejectedException e) {
+ throw new IOException(e);
+ }
+ }
+
+ public void addTable(Text tableName) throws AccumuloException, AccumuloSecurityException {
+ if (simulate) {
+ log.info("Simulating adding table: " + tableName);
+ return;
+ }
+
+ log.debug("Adding table: " + tableName);
+ BatchWriter bw = null;
+ String table = tableName.toString();
+
+ if (createTables && !conn.tableOperations().exists(table)) {
+ try {
+ conn.tableOperations().create(table);
+ } catch (AccumuloSecurityException e) {
+ log.error("Accumulo security violation creating " + table, e);
+ throw e;
+ } catch (TableExistsException e) {
+ // Shouldn't happen
+ }
+ }
+
+ try {
+ bw = mtbw.getBatchWriter(table);
+ } catch (TableNotFoundException e) {
+ log.error("Accumulo table " + table + " doesn't exist and cannot be created.", e);
+ throw new AccumuloException(e);
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (AccumuloSecurityException e) {
+ throw e;
+ }
+
+ if (bw != null)
+ bws.put(tableName, bw);
+ }
+
+ private int printMutation(Text table, Mutation m) {
+ if (log.isTraceEnabled()) {
+ log.trace(String.format("Table %s row key: %s", table, hexDump(m.getRow())));
+ for (ColumnUpdate cu : m.getUpdates()) {
+ log.trace(String.format("Table %s column: %s:%s", table, hexDump(cu.getColumnFamily()), hexDump(cu.getColumnQualifier())));
+ log.trace(String.format("Table %s security: %s", table, new ColumnVisibility(cu.getColumnVisibility()).toString()));
+ log.trace(String.format("Table %s value: %s", table, hexDump(cu.getValue())));
+ }
+ }
+ return m.getUpdates().size();
+ }
+
+ private String hexDump(byte[] ba) {
+ StringBuilder sb = new StringBuilder();
+ for (byte b : ba) {
+ if ((b > 0x20) && (b < 0x7e))
+ sb.append((char) b);
+ else
+ sb.append(String.format("x%02x", b));
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public void close(TaskAttemptContext attempt) throws IOException, InterruptedException {
+ log.debug("mutations written: " + mutCount + ", values written: " + valCount);
+ if (simulate)
+ return;
+
+ try {
+ mtbw.close();
+ } catch (MutationsRejectedException e) {
+ if (e.getAuthorizationFailuresMap().size() >= 0) {
+ HashMap<String,Set<SecurityErrorCode>> tables = new HashMap<String,Set<SecurityErrorCode>>();
+ for (Entry<KeyExtent,Set<SecurityErrorCode>> ke : e.getAuthorizationFailuresMap().entrySet()) {
+ Set<SecurityErrorCode> secCodes = tables.get(ke.getKey().getTableId().toString());
+ if (secCodes == null) {
+ secCodes = new HashSet<SecurityErrorCode>();
+ tables.put(ke.getKey().getTableId().toString(), secCodes);
+ }
+ secCodes.addAll(ke.getValue());
+ }
+
+ log.error("Not authorized to write to tables : " + tables);
+ }
+
+ if (e.getConstraintViolationSummaries().size() > 0) {
+ log.error("Constraint violations : " + e.getConstraintViolationSummaries().size());
+ }
+ }
+ }
+ }
+
+ @Override
+ public void checkOutputSpecs(JobContext job) throws IOException {
+ if (!isConnectorInfoSet(job))
+ throw new IOException("Connector info has not been set.");
+ try {
+ // if the instance isn't configured, it will complain here
+ Connector c = getInstance(job).getConnector(getPrincipal(job), CredentialHelper.extractToken(getTokenClass(job), getToken(job)));
+ if (!c.securityOperations().authenticateUser(getPrincipal(job), CredentialHelper.extractToken(getTokenClass(job), getToken(job))))
+ throw new IOException("Unable to authenticate user");
+ } catch (AccumuloException e) {
+ throw new IOException(e);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public OutputCommitter getOutputCommitter(TaskAttemptContext context) {
+ return new NullOutputFormat<Text,Mutation>().getOutputCommitter(context);
+ }
+
+ @Override
+ public RecordWriter<Text,Mutation> getRecordWriter(TaskAttemptContext attempt) throws IOException {
+ try {
+ return new AccumuloRecordWriter(attempt);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ // ----------------------------------------------------------------------------------------------------
+ // Everything below this line is deprecated and should go away in future versions
+ // ----------------------------------------------------------------------------------------------------
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setConnectorInfo(Job, String, AuthenticationToken)}, {@link #setCreateTables(Job, boolean)}, and
+ * {@link #setDefaultTableName(Job, String)} instead.
+ */
+ @Deprecated
+ public static void setOutputInfo(Configuration conf, String user, byte[] passwd, boolean createTables, String defaultTable) {
+ try {
+ OutputConfigurator.setConnectorInfo(CLASS, conf, user, new PasswordToken(passwd));
+ } catch (AccumuloSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ OutputConfigurator.setCreateTables(CLASS, conf, createTables);
+ OutputConfigurator.setDefaultTableName(CLASS, conf, defaultTable);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setZooKeeperInstance(Job, String, String)} instead.
+ */
+ @Deprecated
+ public static void setZooKeeperInstance(Configuration conf, String instanceName, String zooKeepers) {
+ OutputConfigurator.setZooKeeperInstance(CLASS, conf, instanceName, zooKeepers);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setMockInstance(Job, String)} instead.
+ */
+ @Deprecated
+ public static void setMockInstance(Configuration conf, String instanceName) {
+ OutputConfigurator.setMockInstance(CLASS, conf, instanceName);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setBatchWriterOptions(Job, BatchWriterConfig)} instead.
+ */
+ @Deprecated
+ public static void setMaxMutationBufferSize(Configuration conf, long numberOfBytes) {
+ BatchWriterConfig bwConfig = OutputConfigurator.getBatchWriterOptions(CLASS, conf);
+ bwConfig.setMaxMemory(numberOfBytes);
+ OutputConfigurator.setBatchWriterOptions(CLASS, conf, bwConfig);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setBatchWriterOptions(Job, BatchWriterConfig)} instead.
+ */
+ @Deprecated
+ public static void setMaxLatency(Configuration conf, int numberOfMilliseconds) {
+ BatchWriterConfig bwConfig = OutputConfigurator.getBatchWriterOptions(CLASS, conf);
+ bwConfig.setMaxLatency(numberOfMilliseconds, TimeUnit.MILLISECONDS);
+ OutputConfigurator.setBatchWriterOptions(CLASS, conf, bwConfig);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setBatchWriterOptions(Job, BatchWriterConfig)} instead.
+ */
+ @Deprecated
+ public static void setMaxWriteThreads(Configuration conf, int numberOfThreads) {
+ BatchWriterConfig bwConfig = OutputConfigurator.getBatchWriterOptions(CLASS, conf);
+ bwConfig.setMaxWriteThreads(numberOfThreads);
+ OutputConfigurator.setBatchWriterOptions(CLASS, conf, bwConfig);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setLogLevel(Job, Level)} instead.
+ */
+ @Deprecated
+ public static void setLogLevel(Configuration conf, Level level) {
+ OutputConfigurator.setLogLevel(CLASS, conf, level);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setSimulationMode(Job, boolean)} instead.
+ */
+ @Deprecated
+ public static void setSimulationMode(Configuration conf) {
+ OutputConfigurator.setSimulationMode(CLASS, conf, true);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getToken(JobContext)} instead.
+ */
+ @Deprecated
+ protected static String getPrincipal(Configuration conf) {
+ return OutputConfigurator.getPrincipal(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getToken(JobContext)} instead.
+ */
+ @Deprecated
+ protected static byte[] getToken(Configuration conf) {
+ return OutputConfigurator.getToken(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #canCreateTables(JobContext)} instead.
+ */
+ @Deprecated
+ protected static boolean canCreateTables(Configuration conf) {
+ return OutputConfigurator.canCreateTables(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getDefaultTableName(JobContext)} instead.
+ */
+ @Deprecated
+ protected static String getDefaultTableName(Configuration conf) {
+ return OutputConfigurator.getDefaultTableName(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getInstance(JobContext)} instead.
+ */
+ @Deprecated
+ protected static Instance getInstance(Configuration conf) {
+ return OutputConfigurator.getInstance(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getBatchWriterOptions(JobContext)} instead.
+ */
+ @Deprecated
+ protected static long getMaxMutationBufferSize(Configuration conf) {
+ return OutputConfigurator.getBatchWriterOptions(CLASS, conf).getMaxMemory();
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getBatchWriterOptions(JobContext)} instead.
+ */
+ @Deprecated
+ protected static int getMaxLatency(Configuration conf) {
+ return (int) OutputConfigurator.getBatchWriterOptions(CLASS, conf).getMaxLatency(TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getBatchWriterOptions(JobContext)} instead.
+ */
+ @Deprecated
+ protected static int getMaxWriteThreads(Configuration conf) {
+ return OutputConfigurator.getBatchWriterOptions(CLASS, conf).getMaxWriteThreads();
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getLogLevel(JobContext)} instead.
+ */
+ @Deprecated
+ protected static Level getLogLevel(Configuration conf) {
+ return OutputConfigurator.getLogLevel(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getSimulationMode(JobContext)} instead.
+ */
+ @Deprecated
+ protected static boolean getSimulationMode(Configuration conf) {
+ return OutputConfigurator.getSimulationMode(CLASS, conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
index 0e9444d,0000000..710c565
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
@@@ -1,1337 -1,0 +1,1336 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Method;
+import java.net.InetAddress;
+import java.net.URLDecoder;
+import java.net.URLEncoder;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.StringTokenizer;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientSideIteratorScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.RowIterator;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableDeletedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.impl.OfflineScanner;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.TabletLocator;
+import org.apache.accumulo.core.client.mapreduce.lib.util.InputConfigurator;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.VersioningIterator;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * This abstract {@link InputFormat} class allows MapReduce jobs to use Accumulo as the source of K,V pairs.
+ * <p>
+ * Subclasses must implement a {@link #createRecordReader(InputSplit, TaskAttemptContext)} to provide a {@link RecordReader} for K,V.
+ * <p>
+ * A static base class, RecordReaderBase, is provided to retrieve Accumulo {@link Key}/{@link Value} pairs, but one must implement its
+ * {@link RecordReaderBase#nextKeyValue()} to transform them to the desired generic types K,V.
+ * <p>
+ * See {@link AccumuloInputFormat} for an example implementation.
+ */
+public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
+
+ private static final Class<?> CLASS = AccumuloInputFormat.class;
+ protected static final Logger log = Logger.getLogger(CLASS);
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
+ * conversion to a string, and is not intended to be secure.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission)
+ * @param token
+ * the user's password
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public static void setConnectorInfo(Job job, String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ InputConfigurator.setConnectorInfo(CLASS, job.getConfiguration(), principal, token);
+ }
+
+ /**
+ * Determines if the connector has been configured.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the connector has been configured, false otherwise
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static Boolean isConnectorInfoSet(JobContext context) {
+ return InputConfigurator.isConnectorInfoSet(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static String getPrincipal(JobContext context) {
+ return InputConfigurator.getPrincipal(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Gets the serialized token class from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static String getTokenClass(JobContext context) {
+ return InputConfigurator.getTokenClass(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Gets the password from the configuration. WARNING: The password is stored in the Configuration and shared with all MapReduce tasks; It is BASE64 encoded to
+ * provide a charset safe conversion to a string, and is not intended to be secure.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the decoded user password
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static byte[] getToken(JobContext context) {
+ return InputConfigurator.getToken(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Configures a {@link ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @param zooKeepers
+ * a comma-separated list of zookeeper servers
+ * @since 1.5.0
+ */
+ public static void setZooKeeperInstance(Job job, String instanceName, String zooKeepers) {
+ InputConfigurator.setZooKeeperInstance(CLASS, job.getConfiguration(), instanceName, zooKeepers);
+ }
+
+ /**
+ * Configures a {@link MockInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @since 1.5.0
+ */
+ public static void setMockInstance(Job job, String instanceName) {
+ InputConfigurator.setMockInstance(CLASS, job.getConfiguration(), instanceName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link Instance} based on the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return an Accumulo instance
+ * @since 1.5.0
+ * @see #setZooKeeperInstance(Job, String, String)
+ * @see #setMockInstance(Job, String)
+ */
+ protected static Instance getInstance(JobContext context) {
+ return InputConfigurator.getInstance(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param level
+ * the logging level
+ * @since 1.5.0
+ */
+ public static void setLogLevel(Job job, Level level) {
+ InputConfigurator.setLogLevel(CLASS, job.getConfiguration(), level);
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the log level
+ * @since 1.5.0
+ * @see #setLogLevel(Job, Level)
+ */
+ protected static Level getLogLevel(JobContext context) {
+ return InputConfigurator.getLogLevel(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Sets the name of the input table, over which this job will scan.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.5.0
+ */
+ public static void setInputTableName(Job job, String tableName) {
+ InputConfigurator.setInputTableName(CLASS, job.getConfiguration(), tableName);
+ }
+
+ /**
+ * Gets the table name from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the table name
+ * @since 1.5.0
+ * @see #setInputTableName(Job, String)
+ */
+ protected static String getInputTableName(JobContext context) {
+ return InputConfigurator.getInputTableName(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Sets the {@link Authorizations} used to scan. Must be a subset of the user's authorization. Defaults to the empty set.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param auths
+ * the user's authorizations
+ * @since 1.5.0
+ */
+ public static void setScanAuthorizations(Job job, Authorizations auths) {
+ InputConfigurator.setScanAuthorizations(CLASS, job.getConfiguration(), auths);
+ }
+
+ /**
+ * Gets the authorizations to set for the scans from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the Accumulo scan authorizations
+ * @since 1.5.0
+ * @see #setScanAuthorizations(Job, Authorizations)
+ */
+ protected static Authorizations getScanAuthorizations(JobContext context) {
+ return InputConfigurator.getScanAuthorizations(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Sets the input ranges to scan for this job. If not set, the entire table will be scanned.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param ranges
+ * the ranges that will be mapped over
+ * @since 1.5.0
+ */
+ public static void setRanges(Job job, Collection<Range> ranges) {
+ InputConfigurator.setRanges(CLASS, job.getConfiguration(), ranges);
+ }
+
+ /**
+ * Gets the ranges to scan over from a job.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the ranges
+ * @throws IOException
+ * if the ranges have been encoded improperly
+ * @since 1.5.0
+ * @see #setRanges(Job, Collection)
+ */
+ protected static List<Range> getRanges(JobContext context) throws IOException {
+ return InputConfigurator.getRanges(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Restricts the columns that will be mapped over for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param columnFamilyColumnQualifierPairs
+ * a pair of {@link Text} objects corresponding to column family and column qualifier. If the column qualifier is null, the entire column family is
+ * selected. An empty set is the default and is equivalent to scanning the all columns.
+ * @since 1.5.0
+ */
+ public static void fetchColumns(Job job, Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
+ InputConfigurator.fetchColumns(CLASS, job.getConfiguration(), columnFamilyColumnQualifierPairs);
+ }
+
+ /**
+ * Gets the columns to be mapped over from this job.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return a set of columns
+ * @since 1.5.0
+ * @see #fetchColumns(Job, Collection)
+ */
+ protected static Set<Pair<Text,Text>> getFetchedColumns(JobContext context) {
+ return InputConfigurator.getFetchedColumns(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Encode an iterator on the input for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param cfg
+ * the configuration of the iterator
+ * @since 1.5.0
+ */
+ public static void addIterator(Job job, IteratorSetting cfg) {
+ InputConfigurator.addIterator(CLASS, job.getConfiguration(), cfg);
+ }
+
+ /**
+ * Gets a list of the iterator settings (for iterators to apply to a scanner) from this configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return a list of iterators
+ * @since 1.5.0
+ * @see #addIterator(Job, IteratorSetting)
+ */
+ protected static List<IteratorSetting> getIterators(JobContext context) {
+ return InputConfigurator.getIterators(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Controls the automatic adjustment of ranges for this job. This feature merges overlapping ranges, then splits them to align with tablet boundaries.
+ * Disabling this feature will cause exactly one Map task to be created for each specified range. The default setting is enabled. *
+ *
+ * <p>
+ * By default, this feature is <b>enabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @see #setRanges(Job, Collection)
+ * @since 1.5.0
+ */
+ public static void setAutoAdjustRanges(Job job, boolean enableFeature) {
+ InputConfigurator.setAutoAdjustRanges(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has auto-adjust ranges enabled.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return false if the feature is disabled, true otherwise
+ * @since 1.5.0
+ * @see #setAutoAdjustRanges(Job, boolean)
+ */
+ protected static boolean getAutoAdjustRanges(JobContext context) {
+ return InputConfigurator.getAutoAdjustRanges(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Controls the use of the {@link IsolatedScanner} in this job.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setScanIsolation(Job job, boolean enableFeature) {
+ InputConfigurator.setScanIsolation(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has isolation enabled.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setScanIsolation(Job, boolean)
+ */
+ protected static boolean isIsolated(JobContext context) {
+ return InputConfigurator.isIsolated(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Controls the use of the {@link ClientSideIteratorScanner} in this job. Enabling this feature will cause the iterator stack to be constructed within the Map
+ * task, rather than within the Accumulo TServer. To use this feature, all classes needed for those iterators must be available on the classpath for the task.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setLocalIterators(Job job, boolean enableFeature) {
+ InputConfigurator.setLocalIterators(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration uses local iterators.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setLocalIterators(Job, boolean)
+ */
+ protected static boolean usesLocalIterators(JobContext context) {
+ return InputConfigurator.usesLocalIterators(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * <p>
+ * Enable reading offline tables. By default, this feature is disabled and only online tables are scanned. This will make the map reduce job directly read the
+ * table's files. If the table is not offline, then the job will fail. If the table comes online during the map reduce job, it is likely that the job will
+ * fail.
+ *
+ * <p>
+ * To use this option, the map reduce user will need access to read the Accumulo directory in HDFS.
+ *
+ * <p>
+ * Reading the offline table will create the scan time iterator stack in the map process. So any iterators that are configured for the table will need to be
+ * on the mapper's classpath.
+ *
+ * <p>
+ * One way to use this feature is to clone a table, take the clone offline, and use the clone as the input table for a map reduce job. If you plan to map
+ * reduce over the data many times, it may be better to the compact the table, clone it, take it offline, and use the clone for all map reduce jobs. The
+ * reason to do this is that compaction will reduce each tablet in the table to one file, and it is faster to read from one file.
+ *
+ * <p>
+ * There are two possible advantages to reading a tables file directly out of HDFS. First, you may see better read performance. Second, it will support
+ * speculative execution better. When reading an online table speculative execution can put more load on an already slow tablet server.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setOfflineTableScan(Job job, boolean enableFeature) {
+ InputConfigurator.setOfflineTableScan(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has the offline table scan feature enabled.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setOfflineTableScan(Job, boolean)
+ */
+ protected static boolean isOfflineScan(JobContext context) {
+ return InputConfigurator.isOfflineScan(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Initializes an Accumulo {@link TabletLocator} based on the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return an Accumulo tablet locator
+ * @throws TableNotFoundException
+ * if the table name set on the configuration doesn't exist
+ * @since 1.5.0
+ */
+ protected static TabletLocator getTabletLocator(JobContext context) throws TableNotFoundException {
+ return InputConfigurator.getTabletLocator(CLASS, getConfiguration(context));
+ }
+
+ // InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext job)
+ /**
+ * Check whether a configuration is fully configured to be used with an Accumulo {@link org.apache.hadoop.mapreduce.InputFormat}.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @throws IOException
+ * if the context is improperly configured
+ * @since 1.5.0
+ */
+ protected static void validateOptions(JobContext context) throws IOException {
+ InputConfigurator.validateOptions(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * An abstract base class to be used to create {@link RecordReader} instances that convert from Accumulo {@link Key}/{@link Value} pairs to the user's K/V
+ * types.
+ *
+ * Subclasses must implement {@link #nextKeyValue()} and use it to update the following variables:
+ * <ul>
+ * <li>K {@link #currentK}</li>
+ * <li>V {@link #currentV}</li>
+ * <li>Key {@link #currentKey} (used for progress reporting)</li>
+ * <li>int {@link #numKeysRead} (used for progress reporting)</li>
+ * </ul>
+ */
+ protected abstract static class RecordReaderBase<K,V> extends RecordReader<K,V> {
+ protected long numKeysRead;
+ protected Iterator<Entry<Key,Value>> scannerIterator;
+ protected org.apache.accumulo.core.client.mapreduce.RangeInputSplit split;
+
+ /**
+ * Apply the configured iterators from the configuration to the scanner.
+ *
+ * @param scanner
+ * the scanner to configure
+ */
+ protected void setupIterators(List<IteratorSetting> iterators, Scanner scanner) {
+ for (IteratorSetting iterator : iterators) {
+ scanner.addScanIterator(iterator);
+ }
+ }
+
+ /**
+ * Initialize a scanner over the given input split using this task attempt configuration.
+ */
+ @Override
+ public void initialize(InputSplit inSplit, TaskAttemptContext attempt) throws IOException {
+ Scanner scanner;
+ split = (org.apache.accumulo.core.client.mapreduce.RangeInputSplit) inSplit;
+ log.debug("Initializing input split: " + split.getRange());
+
+ Instance instance = split.getInstance();
+ if (null == instance) {
+ instance = getInstance(attempt);
+ }
+
+ String principal = split.getPrincipal();
+ if (null == principal) {
+ principal = getPrincipal(attempt);
+ }
+
+ AuthenticationToken token = split.getToken();
+ if (null == token) {
+ String tokenClass = getTokenClass(attempt);
+ byte[] tokenBytes = getToken(attempt);
+ try {
+ token = CredentialHelper.extractToken(tokenClass, tokenBytes);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+ }
+
+ Authorizations authorizations = split.getAuths();
+ if (null == authorizations) {
+ authorizations = getScanAuthorizations(attempt);
+ }
+
+ String table = split.getTable();
+ if (null == table) {
+ table = getInputTableName(attempt);
+ }
+
+ Boolean isOffline = split.isOffline();
+ if (null == isOffline) {
+ isOffline = isOfflineScan(attempt);
+ }
+
+ Boolean isIsolated = split.isIsolatedScan();
+ if (null == isIsolated) {
+ isIsolated = isIsolated(attempt);
+ }
+
+ Boolean usesLocalIterators = split.usesLocalIterators();
+ if (null == usesLocalIterators) {
+ usesLocalIterators = usesLocalIterators(attempt);
+ }
+
+ List<IteratorSetting> iterators = split.getIterators();
+ if (null == iterators) {
+ iterators = getIterators(attempt);
+ }
+
+ Set<Pair<Text,Text>> columns = split.getFetchedColumns();
+ if (null == columns) {
+ columns = getFetchedColumns(attempt);
+ }
+
+ try {
+ log.debug("Creating connector with user: " + principal);
+ Connector conn = instance.getConnector(principal, token);
+ log.debug("Creating scanner for table: " + table);
+ log.debug("Authorizations are: " + authorizations);
+ if (isOffline) {
+ String tokenClass = token.getClass().getCanonicalName();
+ ByteBuffer tokenBuffer = ByteBuffer.wrap(CredentialHelper.toBytes(token));
+ scanner = new OfflineScanner(instance, new TCredentials(principal, tokenClass, tokenBuffer, instance.getInstanceID()), Tables.getTableId(instance,
+ table), authorizations);
+ } else {
+ scanner = conn.createScanner(table, authorizations);
+ }
+ if (isIsolated) {
+ log.info("Creating isolated scanner");
+ scanner = new IsolatedScanner(scanner);
+ }
+ if (usesLocalIterators) {
+ log.info("Using local iterators");
+ scanner = new ClientSideIteratorScanner(scanner);
+ }
+ setupIterators(iterators, scanner);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ // setup a scanner within the bounds of this split
+ for (Pair<Text,Text> c : columns) {
+ if (c.getSecond() != null) {
+ log.debug("Fetching column " + c.getFirst() + ":" + c.getSecond());
+ scanner.fetchColumn(c.getFirst(), c.getSecond());
+ } else {
+ log.debug("Fetching column family " + c.getFirst());
+ scanner.fetchColumnFamily(c.getFirst());
+ }
+ }
+
+ scanner.setRange(split.getRange());
+
+ numKeysRead = 0;
+
+ // do this last after setting all scanner options
+ scannerIterator = scanner.iterator();
+ }
+
+ @Override
+ public void close() {}
+
+ @Override
+ public float getProgress() throws IOException {
+ if (numKeysRead > 0 && currentKey == null)
+ return 1.0f;
+ return split.getProgress(currentKey);
+ }
+
+ protected K currentK = null;
+ protected V currentV = null;
+ protected Key currentKey = null;
+ protected Value currentValue = null;
+
+ @Override
+ public K getCurrentKey() throws IOException, InterruptedException {
+ return currentK;
+ }
+
+ @Override
+ public V getCurrentValue() throws IOException, InterruptedException {
+ return currentV;
+ }
+ }
+
+ Map<String,Map<KeyExtent,List<Range>>> binOfflineTable(JobContext context, String tableName, List<Range> ranges) throws TableNotFoundException,
+ AccumuloException, AccumuloSecurityException {
+
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+
+ Instance instance = getInstance(context);
+ Connector conn = instance.getConnector(getPrincipal(context), CredentialHelper.extractToken(getTokenClass(context), getToken(context)));
+ String tableId = Tables.getTableId(instance, tableName);
+
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ Tables.clearCache(instance);
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ throw new AccumuloException("Table is online " + tableName + "(" + tableId + ") cannot scan table in offline mode ");
+ }
+ }
+
+ for (Range range : ranges) {
+ Text startRow;
+
+ if (range.getStartKey() != null)
+ startRow = range.getStartKey().getRow();
+ else
+ startRow = new Text();
+
+ Range metadataRange = new Range(new KeyExtent(new Text(tableId), startRow, null).getMetadataEntry(), true, null, false);
+ Scanner scanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ Constants.METADATA_PREV_ROW_COLUMN.fetch(scanner);
+ scanner.fetchColumnFamily(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY);
+ scanner.setRange(metadataRange);
+
+ RowIterator rowIter = new RowIterator(scanner);
+
+ KeyExtent lastExtent = null;
+
+ while (rowIter.hasNext()) {
+ Iterator<Entry<Key,Value>> row = rowIter.next();
+ String last = "";
+ KeyExtent extent = null;
+ String location = null;
+
+ while (row.hasNext()) {
+ Entry<Key,Value> entry = row.next();
+ Key key = entry.getKey();
+
+ if (key.getColumnFamily().equals(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY)) {
+ last = entry.getValue().toString();
+ }
+
+ if (key.getColumnFamily().equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY)
+ || key.getColumnFamily().equals(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY)) {
+ location = entry.getValue().toString();
+ }
+
+ if (Constants.METADATA_PREV_ROW_COLUMN.hasColumns(key)) {
+ extent = new KeyExtent(key.getRow(), entry.getValue());
+ }
+
+ }
+
+ if (location != null)
+ return null;
+
+ if (!extent.getTableId().toString().equals(tableId)) {
+ throw new AccumuloException("Saw unexpected table Id " + tableId + " " + extent);
+ }
+
+ if (lastExtent != null && !extent.isPreviousExtent(lastExtent)) {
+ throw new AccumuloException(" " + lastExtent + " is not previous extent " + extent);
+ }
+
+ Map<KeyExtent,List<Range>> tabletRanges = binnedRanges.get(last);
+ if (tabletRanges == null) {
+ tabletRanges = new HashMap<KeyExtent,List<Range>>();
+ binnedRanges.put(last, tabletRanges);
+ }
+
+ List<Range> rangeList = tabletRanges.get(extent);
+ if (rangeList == null) {
+ rangeList = new ArrayList<Range>();
+ tabletRanges.put(extent, rangeList);
+ }
+
+ rangeList.add(range);
+
+ if (extent.getEndRow() == null || range.afterEndKey(new Key(extent.getEndRow()).followingKey(PartialKey.ROW))) {
+ break;
+ }
+
+ lastExtent = extent;
+ }
+
+ }
+
+ return binnedRanges;
+ }
+
+ /**
+ * Read the metadata table to get tablets and match up ranges to them.
+ */
+ @Override
+ public List<InputSplit> getSplits(JobContext context) throws IOException {
+ Level logLevel = getLogLevel(context);
+ log.setLevel(logLevel);
+
+ validateOptions(context);
+
+ String tableName = getInputTableName(context);
+ boolean autoAdjust = getAutoAdjustRanges(context);
+ List<Range> ranges = autoAdjust ? Range.mergeOverlapping(getRanges(context)) : getRanges(context);
+ Instance instance = getInstance(context);
+ boolean offline = isOfflineScan(context);
+ boolean isolated = isIsolated(context);
+ boolean localIterators = usesLocalIterators(context);
+ boolean mockInstance = (null != instance && MockInstance.class.equals(instance.getClass()));
+ Set<Pair<Text,Text>> fetchedColumns = getFetchedColumns(context);
+ Authorizations auths = getScanAuthorizations(context);
+ String principal = getPrincipal(context);
+ String tokenClass = getTokenClass(context);
+ byte[] tokenBytes = getToken(context);
+
+ AuthenticationToken token;
+ try {
+ token = CredentialHelper.extractToken(tokenClass, tokenBytes);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+
+ List<IteratorSetting> iterators = getIterators(context);
+
+ if (ranges.isEmpty()) {
+ ranges = new ArrayList<Range>(1);
+ ranges.add(new Range());
+ }
+
+ // get the metadata information for these ranges
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+ TabletLocator tl;
+ try {
+ if (isOfflineScan(context)) {
+ binnedRanges = binOfflineTable(context, tableName, ranges);
+ while (binnedRanges == null) {
+ // Some tablets were still online, try again
+ UtilWaitThread.sleep(100 + (int) (Math.random() * 100)); // sleep randomly between 100 and 200 ms
+ binnedRanges = binOfflineTable(context, tableName, ranges);
+ }
+ } else {
+ String tableId = null;
+ tl = getTabletLocator(context);
+ // its possible that the cache could contain complete, but old information about a tables tablets... so clear it
+ tl.invalidateCache();
+ while (!tl.binRanges(ranges, binnedRanges, new TCredentials(principal, tokenClass, ByteBuffer.wrap(tokenBytes), instance.getInstanceID())).isEmpty()) {
+ if (!(instance instanceof MockInstance)) {
+ if (tableId == null)
+ tableId = Tables.getTableId(instance, tableName);
+ if (!Tables.exists(instance, tableId))
+ throw new TableDeletedException(tableId);
+ if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
+ throw new TableOfflineException(instance, tableId);
+ }
+ binnedRanges.clear();
+ log.warn("Unable to locate bins for specified ranges. Retrying.");
+ UtilWaitThread.sleep(100 + (int) (Math.random() * 100)); // sleep randomly between 100 and 200 ms
+ tl.invalidateCache();
+ }
+ }
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ ArrayList<InputSplit> splits = new ArrayList<InputSplit>(ranges.size());
+ HashMap<Range,ArrayList<String>> splitsToAdd = null;
+
+ if (!autoAdjust)
+ splitsToAdd = new HashMap<Range,ArrayList<String>>();
+
+ HashMap<String,String> hostNameCache = new HashMap<String,String>();
+
+ for (Entry<String,Map<KeyExtent,List<Range>>> tserverBin : binnedRanges.entrySet()) {
+ String ip = tserverBin.getKey().split(":", 2)[0];
+ String location = hostNameCache.get(ip);
+ if (location == null) {
+ InetAddress inetAddress = InetAddress.getByName(ip);
+ location = inetAddress.getHostName();
+ hostNameCache.put(ip, location);
+ }
+
+ for (Entry<KeyExtent,List<Range>> extentRanges : tserverBin.getValue().entrySet()) {
+ Range ke = extentRanges.getKey().toDataRange();
+ for (Range r : extentRanges.getValue()) {
+ if (autoAdjust) {
+ // divide ranges into smaller ranges, based on the tablets
+ splits.add(new org.apache.accumulo.core.client.mapreduce.RangeInputSplit(ke.clip(r), new String[] {location}));
+ } else {
+ // don't divide ranges
+ ArrayList<String> locations = splitsToAdd.get(r);
+ if (locations == null)
+ locations = new ArrayList<String>(1);
+ locations.add(location);
+ splitsToAdd.put(r, locations);
+ }
+ }
+ }
+ }
+
+ if (!autoAdjust)
+ for (Entry<Range,ArrayList<String>> entry : splitsToAdd.entrySet())
+ splits.add(new org.apache.accumulo.core.client.mapreduce.RangeInputSplit(entry.getKey(), entry.getValue().toArray(new String[0])));
+
+ for (InputSplit inputSplit : splits) {
+ org.apache.accumulo.core.client.mapreduce.RangeInputSplit split = (org.apache.accumulo.core.client.mapreduce.RangeInputSplit) inputSplit;
+
+ split.setTable(tableName);
+ split.setOffline(offline);
+ split.setIsolatedScan(isolated);
+ split.setUsesLocalIterators(localIterators);
+ split.setMockInstance(mockInstance);
+ split.setFetchedColumns(fetchedColumns);
+ split.setPrincipal(principal);
+ split.setToken(token);
+ split.setInstanceName(instance.getInstanceName());
+ split.setZooKeepers(instance.getZooKeepers());
+ split.setAuths(auths);
+ split.setIterators(iterators);
+ split.setLogLevel(logLevel);
+ }
+
+ return splits;
+ }
+
+ // ----------------------------------------------------------------------------------------------------
+ // Everything below this line is deprecated and should go away in future versions
+ // ----------------------------------------------------------------------------------------------------
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setScanIsolation(Job, boolean)} instead.
+ */
+ @Deprecated
+ public static void setIsolated(Configuration conf, boolean enable) {
+ InputConfigurator.setScanIsolation(CLASS, conf, enable);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setLocalIterators(Job, boolean)} instead.
+ */
+ @Deprecated
+ public static void setLocalIterators(Configuration conf, boolean enable) {
+ InputConfigurator.setLocalIterators(CLASS, conf, enable);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setConnectorInfo(Job, String, AuthenticationToken)}, {@link #setInputTableName(Job, String)}, and
+ * {@link #setScanAuthorizations(Job, Authorizations)} instead.
+ */
+ @Deprecated
+ public static void setInputInfo(Configuration conf, String user, byte[] passwd, String table, Authorizations auths) {
+ try {
+ InputConfigurator.setConnectorInfo(CLASS, conf, user, new PasswordToken(passwd));
+ } catch (AccumuloSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ InputConfigurator.setInputTableName(CLASS, conf, table);
+ InputConfigurator.setScanAuthorizations(CLASS, conf, auths);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setZooKeeperInstance(Job, String, String)} instead.
+ */
+ @Deprecated
+ public static void setZooKeeperInstance(Configuration conf, String instanceName, String zooKeepers) {
+ InputConfigurator.setZooKeeperInstance(CLASS, conf, instanceName, zooKeepers);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setMockInstance(Job, String)} instead.
+ */
+ @Deprecated
+ public static void setMockInstance(Configuration conf, String instanceName) {
+ InputConfigurator.setMockInstance(CLASS, conf, instanceName);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setRanges(Job, Collection)} instead.
+ */
+ @Deprecated
+ public static void setRanges(Configuration conf, Collection<Range> ranges) {
+ InputConfigurator.setRanges(CLASS, conf, ranges);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setAutoAdjustRanges(Job, boolean)} instead.
+ */
+ @Deprecated
+ public static void disableAutoAdjustRanges(Configuration conf) {
+ InputConfigurator.setAutoAdjustRanges(CLASS, conf, false);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #addIterator(Job, IteratorSetting)} to add the {@link VersioningIterator} instead.
+ */
+ @Deprecated
+ public static void setMaxVersions(Configuration conf, int maxVersions) throws IOException {
+ IteratorSetting vers = new IteratorSetting(1, "vers", VersioningIterator.class);
+ try {
+ VersioningIterator.setMaxVersions(vers, maxVersions);
+ } catch (IllegalArgumentException e) {
+ throw new IOException(e);
+ }
+ InputConfigurator.addIterator(CLASS, conf, vers);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setOfflineTableScan(Job, boolean)} instead.
+ */
+ @Deprecated
+ public static void setScanOffline(Configuration conf, boolean scanOff) {
+ InputConfigurator.setOfflineTableScan(CLASS, conf, scanOff);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #fetchColumns(Job, Collection)} instead.
+ */
+ @Deprecated
+ public static void fetchColumns(Configuration conf, Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
+ InputConfigurator.fetchColumns(CLASS, conf, columnFamilyColumnQualifierPairs);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setLogLevel(Job, Level)} instead.
+ */
+ @Deprecated
+ public static void setLogLevel(Configuration conf, Level level) {
+ InputConfigurator.setLogLevel(CLASS, conf, level);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #addIterator(Job, IteratorSetting)} instead.
+ */
+ @Deprecated
+ public static void addIterator(Configuration conf, IteratorSetting cfg) {
+ InputConfigurator.addIterator(CLASS, conf, cfg);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #isIsolated(JobContext)} instead.
+ */
+ @Deprecated
+ protected static boolean isIsolated(Configuration conf) {
+ return InputConfigurator.isIsolated(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #usesLocalIterators(JobContext)} instead.
+ */
+ @Deprecated
+ protected static boolean usesLocalIterators(Configuration conf) {
+ return InputConfigurator.usesLocalIterators(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getPrincipal(JobContext)} instead.
+ */
+ @Deprecated
+ protected static String getPrincipal(Configuration conf) {
+ return InputConfigurator.getPrincipal(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getToken(JobContext)} instead.
+ */
+ @Deprecated
+ protected static byte[] getToken(Configuration conf) {
+ return InputConfigurator.getToken(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getInputTableName(JobContext)} instead.
+ */
+ @Deprecated
+ protected static String getTablename(Configuration conf) {
+ return InputConfigurator.getInputTableName(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getScanAuthorizations(JobContext)} instead.
+ */
+ @Deprecated
+ protected static Authorizations getAuthorizations(Configuration conf) {
+ return InputConfigurator.getScanAuthorizations(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getInstance(JobContext)} instead.
+ */
+ @Deprecated
+ protected static Instance getInstance(Configuration conf) {
+ return InputConfigurator.getInstance(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getTabletLocator(JobContext)} instead.
+ */
+ @Deprecated
+ protected static TabletLocator getTabletLocator(Configuration conf) throws TableNotFoundException {
+ return InputConfigurator.getTabletLocator(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getRanges(JobContext)} instead.
+ */
+ @Deprecated
+ protected static List<Range> getRanges(Configuration conf) throws IOException {
+ return InputConfigurator.getRanges(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getFetchedColumns(JobContext)} instead.
+ */
+ @Deprecated
+ protected static Set<Pair<Text,Text>> getFetchedColumns(Configuration conf) {
+ return InputConfigurator.getFetchedColumns(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getAutoAdjustRanges(JobContext)} instead.
+ */
+ @Deprecated
+ protected static boolean getAutoAdjustRanges(Configuration conf) {
+ return InputConfigurator.getAutoAdjustRanges(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getLogLevel(JobContext)} instead.
+ */
+ @Deprecated
+ protected static Level getLogLevel(Configuration conf) {
+ return InputConfigurator.getLogLevel(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #validateOptions(JobContext)} instead.
+ */
+ @Deprecated
+ protected static void validateOptions(Configuration conf) throws IOException {
+ InputConfigurator.validateOptions(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #addIterator(Job, IteratorSetting)} to add the {@link VersioningIterator} instead.
+ */
+ @Deprecated
+ protected static int getMaxVersions(Configuration conf) {
+ // This is so convoluted, because the only reason to get the number of maxVersions is to construct the same type of IteratorSetting object we have to
+ // deconstruct to get at this option in the first place, but to preserve correct behavior, this appears necessary.
+ List<IteratorSetting> iteratorSettings = InputConfigurator.getIterators(CLASS, conf);
+ for (IteratorSetting setting : iteratorSettings) {
+ if ("vers".equals(setting.getName()) && 1 == setting.getPriority() && VersioningIterator.class.getName().equals(setting.getIteratorClass())) {
+ if (setting.getOptions().containsKey("maxVersions"))
+ return Integer.parseInt(setting.getOptions().get("maxVersions"));
+ else
+ return -1;
+ }
+ }
+ return -1;
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #isOfflineScan(JobContext)} instead.
+ */
+ @Deprecated
+ protected static boolean isOfflineScan(Configuration conf) {
+ return InputConfigurator.isOfflineScan(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getIterators(JobContext)} instead.
+ */
+ @Deprecated
+ protected static List<AccumuloIterator> getIterators(Configuration conf) {
+ List<IteratorSetting> iteratorSettings = InputConfigurator.getIterators(CLASS, conf);
+ List<AccumuloIterator> deprecatedIterators = new ArrayList<AccumuloIterator>(iteratorSettings.size());
+ for (IteratorSetting setting : iteratorSettings) {
+ AccumuloIterator deprecatedIter = new AccumuloIterator(setting.getPriority() + AccumuloIterator.FIELD_SEP + setting.getIteratorClass()
+ + AccumuloIterator.FIELD_SEP + setting.getName());
+ deprecatedIterators.add(deprecatedIter);
+ }
+ return deprecatedIterators;
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getIterators(JobContext)} instead.
+ */
+ @Deprecated
+ protected static List<AccumuloIteratorOption> getIteratorOptions(Configuration conf) {
+ List<IteratorSetting> iteratorSettings = InputConfigurator.getIterators(CLASS, conf);
+ List<AccumuloIteratorOption> deprecatedIteratorOptions = new ArrayList<AccumuloIteratorOption>(iteratorSettings.size());
+ for (IteratorSetting setting : iteratorSettings) {
+ for (Entry<String,String> opt : setting.getOptions().entrySet()) {
+ String deprecatedOption;
+ try {
+ deprecatedOption = setting.getName() + AccumuloIteratorOption.FIELD_SEP + URLEncoder.encode(opt.getKey(), "UTF-8") + AccumuloIteratorOption.FIELD_SEP
+ + URLEncoder.encode(opt.getValue(), "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ deprecatedIteratorOptions.add(new AccumuloIteratorOption(deprecatedOption));
+ }
+ }
+ return deprecatedIteratorOptions;
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link IteratorSetting} instead.
+ */
+ @Deprecated
+ static class AccumuloIterator {
+
+ private static final String FIELD_SEP = ":";
+
+ private int priority;
+ private String iteratorClass;
+ private String iteratorName;
+
+ public AccumuloIterator(int priority, String iteratorClass, String iteratorName) {
+ this.priority = priority;
+ this.iteratorClass = iteratorClass;
+ this.iteratorName = iteratorName;
+ }
+
+ // Parses out a setting given an string supplied from an earlier toString() call
+ public AccumuloIterator(String iteratorSetting) {
+ // Parse the string to expand the iterator
+ StringTokenizer tokenizer = new StringTokenizer(iteratorSetting, FIELD_SEP);
+ priority = Integer.parseInt(tokenizer.nextToken());
+ iteratorClass = tokenizer.nextToken();
+ iteratorName = tokenizer.nextToken();
+ }
+
+ public int getPriority() {
+ return priority;
+ }
+
+ public String getIteratorClass() {
+ return iteratorClass;
+ }
+
+ public String getIteratorName() {
+ return iteratorName;
+ }
+
+ @Override
+ public String toString() {
+ return priority + FIELD_SEP + iteratorClass + FIELD_SEP + iteratorName;
+ }
+
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link IteratorSetting} instead.
+ */
+ @Deprecated
+ static class AccumuloIteratorOption {
+ private static final String FIELD_SEP = ":";
+
+ private String iteratorName;
+ private String key;
+ private String value;
+
+ public AccumuloIteratorOption(String iteratorName, String key, String value) {
+ this.iteratorName = iteratorName;
+ this.key = key;
+ this.value = value;
+ }
+
+ // Parses out an option given a string supplied from an earlier toString() call
+ public AccumuloIteratorOption(String iteratorOption) {
+ StringTokenizer tokenizer = new StringTokenizer(iteratorOption, FIELD_SEP);
+ this.iteratorName = tokenizer.nextToken();
+ try {
+ this.key = URLDecoder.decode(tokenizer.nextToken(), "UTF-8");
+ this.value = URLDecoder.decode(tokenizer.nextToken(), "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public String getIteratorName() {
+ return iteratorName;
+ }
+
+ public String getKey() {
+ return key;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return iteratorName + FIELD_SEP + URLEncoder.encode(key, "UTF-8") + FIELD_SEP + URLEncoder.encode(value, "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+ // use reflection to pull the Configuration out of the JobContext for Hadoop 1 and Hadoop 2 compatibility
+ static Configuration getConfiguration(JobContext context) {
+ try {
+ Class<?> c = InputFormatBase.class.getClassLoader().loadClass("org.apache.hadoop.mapreduce.JobContext");
+ Method m = c.getMethod("getConfiguration");
+ Object o = m.invoke(context, new Object[0]);
+ return (Configuration) o;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * @deprecated since 1.5.2; Use {@link org.apache.accumulo.core.client.mapreduce.RangeInputSplit} instead.
+ * @see org.apache.accumulo.core.client.mapreduce.RangeInputSplit
+ */
+ @Deprecated
+ public static class RangeInputSplit extends org.apache.accumulo.core.client.mapreduce.RangeInputSplit {
+
+ public RangeInputSplit() {
+ super();
+ }
+
+ public RangeInputSplit(Range range, String[] locations) {
+ super(range, locations);
+ }
+ }
+}
[07/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
index f4408f5,0000000..ae5e395
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
@@@ -1,462 -1,0 +1,456 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Level;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class AccumuloInputFormatTest {
+
+ private static final String PREFIX = AccumuloInputFormatTest.class.getSimpleName();
+
+ /**
+ * Test basic setting & getting of max versions.
+ *
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @Deprecated
+ @Test
+ public void testMaxVersions() throws IOException {
+ Job job = new Job();
+ AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 1);
+ int version = AccumuloInputFormat.getMaxVersions(job.getConfiguration());
+ assertEquals(1, version);
+ }
+
+ /**
+ * Test max versions with an invalid value.
+ *
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @Deprecated
+ @Test(expected = IOException.class)
+ public void testMaxVersionsLessThan1() throws IOException {
+ Job job = new Job();
+ AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 0);
+ }
+
+ /**
+ * Test no max version configured.
- *
- * @throws IOException
+ */
+ @Deprecated
+ @Test
+ public void testNoMaxVersion() throws IOException {
+ Job job = new Job();
+ assertEquals(-1, AccumuloInputFormat.getMaxVersions(job.getConfiguration()));
+ }
+
+ /**
+ * Check that the iterator configuration is getting stored in the Job conf correctly.
- *
- * @throws IOException
+ */
+ @Test
+ public void testSetIterator() throws IOException {
+ Job job = new Job();
+
+ IteratorSetting is = new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator");
+ AccumuloInputFormat.addIterator(job, is);
+ Configuration conf = job.getConfiguration();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ is.write(new DataOutputStream(baos));
+ String iterators = conf.get("AccumuloInputFormat.ScanOpts.Iterators");
+ assertEquals(new String(Base64.encodeBase64(baos.toByteArray())), iterators);
+ }
+
+ @Test
+ public void testAddIterator() throws IOException {
+ Job job = new Job();
+
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", WholeRowIterator.class));
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
+ IteratorSetting iter = new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator");
+ iter.addOption("v1", "1");
+ iter.addOption("junk", "\0omg:!\\xyzzy");
+ AccumuloInputFormat.addIterator(job, iter);
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+
+ // Check the list size
+ assertTrue(list.size() == 3);
+
+ // Walk the list and make sure our settings are correct
+ IteratorSetting setting = list.get(0);
+ assertEquals(1, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.user.WholeRowIterator", setting.getIteratorClass());
+ assertEquals("WholeRow", setting.getName());
+ assertEquals(0, setting.getOptions().size());
+
+ setting = list.get(1);
+ assertEquals(2, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
+ assertEquals("Versions", setting.getName());
+ assertEquals(0, setting.getOptions().size());
+
+ setting = list.get(2);
+ assertEquals(3, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
+ assertEquals("Count", setting.getName());
+ assertEquals(2, setting.getOptions().size());
+ assertEquals("1", setting.getOptions().get("v1"));
+ assertEquals("\0omg:!\\xyzzy", setting.getOptions().get("junk"));
+ }
+
+ /**
+ * Test adding iterator options where the keys and values contain both the FIELD_SEPARATOR character (':') and ITERATOR_SEPARATOR (',') characters. There
+ * should be no exceptions thrown when trying to parse these types of option entries.
+ *
+ * This test makes sure that the expected raw values, as appears in the Job, are equal to what's expected.
+ */
+ @Test
+ public void testIteratorOptionEncoding() throws Throwable {
+ String key = "colon:delimited:key";
+ String value = "comma,delimited,value";
+ IteratorSetting someSetting = new IteratorSetting(1, "iterator", "Iterator.class");
+ someSetting.addOption(key, value);
+ Job job = new Job();
+ AccumuloInputFormat.addIterator(job, someSetting);
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+ assertEquals(1, list.size());
+ assertEquals(1, list.get(0).getOptions().size());
+ assertEquals(list.get(0).getOptions().get(key), value);
+
+ someSetting.addOption(key + "2", value);
+ someSetting.setPriority(2);
+ someSetting.setName("it2");
+ AccumuloInputFormat.addIterator(job, someSetting);
+ list = AccumuloInputFormat.getIterators(job);
+ assertEquals(2, list.size());
+ assertEquals(1, list.get(0).getOptions().size());
+ assertEquals(list.get(0).getOptions().get(key), value);
+ assertEquals(2, list.get(1).getOptions().size());
+ assertEquals(list.get(1).getOptions().get(key), value);
+ assertEquals(list.get(1).getOptions().get(key + "2"), value);
+ }
+
+ /**
+ * Test getting iterator settings for multiple iterators set
- *
- * @throws IOException
+ */
+ @Test
+ public void testGetIteratorSettings() throws IOException {
+ Job job = new Job();
+
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator"));
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator"));
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+
+ // Check the list size
+ assertTrue(list.size() == 3);
+
+ // Walk the list and make sure our settings are correct
+ IteratorSetting setting = list.get(0);
+ assertEquals(1, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.WholeRowIterator", setting.getIteratorClass());
+ assertEquals("WholeRow", setting.getName());
+
+ setting = list.get(1);
+ assertEquals(2, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
+ assertEquals("Versions", setting.getName());
+
+ setting = list.get(2);
+ assertEquals(3, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
+ assertEquals("Count", setting.getName());
+
+ }
+
+ @Test
+ public void testSetRegex() throws IOException {
+ Job job = new Job();
+
+ String regex = ">\"*%<>\'\\";
+
+ IteratorSetting is = new IteratorSetting(50, regex, RegExFilter.class);
+ RegExFilter.setRegexs(is, regex, null, null, null, false);
+ AccumuloInputFormat.addIterator(job, is);
+
+ assertTrue(regex.equals(AccumuloInputFormat.getIterators(job).get(0).getName()));
+ }
+
+ private static AssertionError e1 = null;
+ private static AssertionError e2 = null;
+
+ private static class MRTester extends Configured implements Tool {
+ private static class TestMapper extends Mapper<Key,Value,Key,Value> {
+ Key key = null;
+ int count = 0;
+
+ @Override
+ protected void map(Key k, Value v, Context context) throws IOException, InterruptedException {
+ try {
+ if (key != null)
+ assertEquals(key.getRow().toString(), new String(v.get()));
+ assertEquals(k.getRow(), new Text(String.format("%09x", count + 1)));
+ assertEquals(new String(v.get()), String.format("%09x", count));
+ } catch (AssertionError e) {
+ e1 = e;
+ }
+ key = new Key(k);
+ count++;
+ }
+
+ @Override
+ protected void cleanup(Context context) throws IOException, InterruptedException {
+ try {
+ assertEquals(100, count);
+ } catch (AssertionError e) {
+ e2 = e;
+ }
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+
+ if (args.length != 5) {
+ throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <user> <pass> <table> <instanceName> <inputFormatClass>");
+ }
+
+ String user = args[0];
+ String pass = args[1];
+ String table = args[2];
+ String instanceName = args[3];
+ String inputFormatClassName = args[4];
+ @SuppressWarnings("unchecked")
+ Class<? extends InputFormat<?,?>> inputFormatClass = (Class<? extends InputFormat<?,?>>) Class.forName(inputFormatClassName);
+
+ Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+ job.setJarByClass(this.getClass());
+
+ job.setInputFormatClass(inputFormatClass);
+
+ AccumuloInputFormat.setConnectorInfo(job, user, new PasswordToken(pass));
+ AccumuloInputFormat.setInputTableName(job, table);
+ AccumuloInputFormat.setMockInstance(job, instanceName);
+
+ job.setMapperClass(TestMapper.class);
+ job.setMapOutputKeyClass(Key.class);
+ job.setMapOutputValueClass(Value.class);
+ job.setOutputFormatClass(NullOutputFormat.class);
+
+ job.setNumReduceTasks(0);
+
+ job.waitForCompletion(true);
+
+ return job.isSuccessful() ? 0 : 1;
+ }
+
+ public static int main(String[] args) throws Exception {
+ return ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args);
+ }
+ }
+
+ @Test
+ public void testMap() throws Exception {
+ final String INSTANCE_NAME = PREFIX + "_mapreduce_instance";
+ final String TEST_TABLE_1 = PREFIX + "_mapreduce_table_1";
+
+ MockInstance mockInstance = new MockInstance(INSTANCE_NAME);
+ Connector c = mockInstance.getConnector("root", new PasswordToken(""));
+ c.tableOperations().create(TEST_TABLE_1);
+ BatchWriter bw = c.createBatchWriter(TEST_TABLE_1, new BatchWriterConfig());
+ for (int i = 0; i < 100; i++) {
+ Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
+ m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
+ bw.addMutation(m);
+ }
+ bw.close();
+
+ Assert.assertEquals(0, MRTester.main(new String[] {"root", "", TEST_TABLE_1, INSTANCE_NAME, AccumuloInputFormat.class.getCanonicalName()}));
+ assertNull(e1);
+ assertNull(e2);
+ }
+
+ @Test
+ public void testCorrectRangeInputSplits() throws Exception {
+ Job job = new Job(new Configuration(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+
+ String username = "user", table = "table", instance = "instance";
+ PasswordToken password = new PasswordToken("password");
+ Authorizations auths = new Authorizations("foo");
+ Collection<Pair<Text,Text>> fetchColumns = Collections.singleton(new Pair<Text,Text>(new Text("foo"), new Text("bar")));
+ boolean isolated = true, localIters = true;
+ Level level = Level.WARN;
+
+ Instance inst = new MockInstance(instance);
+ Connector connector = inst.getConnector(username, password);
+ connector.tableOperations().create(table);
+
+ AccumuloInputFormat.setConnectorInfo(job, username, password);
+ AccumuloInputFormat.setInputTableName(job, table);
+ AccumuloInputFormat.setScanAuthorizations(job, auths);
+ AccumuloInputFormat.setMockInstance(job, instance);
+ AccumuloInputFormat.setScanIsolation(job, isolated);
+ AccumuloInputFormat.setLocalIterators(job, localIters);
+ AccumuloInputFormat.fetchColumns(job, fetchColumns);
+ AccumuloInputFormat.setLogLevel(job, level);
+
+ AccumuloInputFormat aif = new AccumuloInputFormat();
+
+ List<InputSplit> splits = aif.getSplits(job);
+
+ Assert.assertEquals(1, splits.size());
+
+ InputSplit split = splits.get(0);
+
+ Assert.assertEquals(RangeInputSplit.class, split.getClass());
+
+ RangeInputSplit risplit = (RangeInputSplit) split;
+
+ Assert.assertEquals(username, risplit.getPrincipal());
+ Assert.assertEquals(table, risplit.getTable());
+ Assert.assertEquals(password, risplit.getToken());
+ Assert.assertEquals(auths, risplit.getAuths());
+ Assert.assertEquals(instance, risplit.getInstanceName());
+ Assert.assertEquals(isolated, risplit.isIsolatedScan());
+ Assert.assertEquals(localIters, risplit.usesLocalIterators());
+ Assert.assertEquals(fetchColumns, risplit.getFetchedColumns());
+ Assert.assertEquals(level, risplit.getLogLevel());
+ }
+
+ static class TestMapper extends Mapper<Key,Value,Key,Value> {
+ Key key = null;
+ int count = 0;
+
+ @Override
+ protected void map(Key k, Value v, Context context) throws IOException, InterruptedException {
+ if (key != null)
+ assertEquals(key.getRow().toString(), new String(v.get()));
+ assertEquals(k.getRow(), new Text(String.format("%09x", count + 1)));
+ assertEquals(new String(v.get()), String.format("%09x", count));
+ key = new Key(k);
+ count++;
+ }
+ }
+
+ @Test
+ public void testPartialInputSplitDelegationToConfiguration() throws Exception {
+ String user = "testPartialInputSplitUser";
+ PasswordToken password = new PasswordToken("");
+
+ MockInstance mockInstance = new MockInstance("testPartialInputSplitDelegationToConfiguration");
+ Connector c = mockInstance.getConnector(user, password);
+ c.tableOperations().create("testtable");
+ BatchWriter bw = c.createBatchWriter("testtable", new BatchWriterConfig());
+ for (int i = 0; i < 100; i++) {
+ Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
+ m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
+ bw.addMutation(m);
+ }
+ bw.close();
+
+ Assert.assertEquals(
+ 0,
+ MRTester.main(new String[] {user, "", "testtable", "testPartialInputSplitDelegationToConfiguration",
+ EmptySplitsAccumuloInputFormat.class.getCanonicalName()}));
+ assertNull(e1);
+ assertNull(e2);
+ }
+
+ @Test
+ public void testPartialFailedInputSplitDelegationToConfiguration() throws Exception {
+ String user = "testPartialFailedInputSplit";
+ PasswordToken password = new PasswordToken("");
+
+ MockInstance mockInstance = new MockInstance("testPartialFailedInputSplitDelegationToConfiguration");
+ Connector c = mockInstance.getConnector(user, password);
+ c.tableOperations().create("testtable");
+ BatchWriter bw = c.createBatchWriter("testtable", new BatchWriterConfig());
+ for (int i = 0; i < 100; i++) {
+ Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
+ m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
+ bw.addMutation(m);
+ }
+ bw.close();
+
+ // We should fail before we even get into the Mapper because we can't make the RecordReader
+ Assert.assertEquals(
+ 1,
+ MRTester.main(new String[] {user, "", "testtable", "testPartialFailedInputSplitDelegationToConfiguration",
+ BadPasswordSplitsAccumuloInputFormat.class.getCanonicalName()}));
+ assertNull(e1);
+ assertNull(e2);
+ }
+
+ @Test
+ public void testEmptyColumnFamily() throws IOException {
+ Job job = new Job();
+ Set<Pair<Text,Text>> cols = new HashSet<Pair<Text,Text>>();
+ cols.add(new Pair<Text,Text>(new Text(""), null));
+ cols.add(new Pair<Text,Text>(new Text("foo"), new Text("bar")));
+ cols.add(new Pair<Text,Text>(new Text(""), new Text("bar")));
+ cols.add(new Pair<Text,Text>(new Text(""), new Text("")));
+ cols.add(new Pair<Text,Text>(new Text("foo"), new Text("")));
+ AccumuloInputFormat.fetchColumns(job, cols);
+ Set<Pair<Text,Text>> setCols = AccumuloInputFormat.getFetchedColumns(job);
+ assertEquals(cols, setCols);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/test/java/org/apache/accumulo/core/util/shell/command/FormatterCommandTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/util/shell/command/FormatterCommandTest.java
index 0eb2653,0000000..6000817
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/util/shell/command/FormatterCommandTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/shell/command/FormatterCommandTest.java
@@@ -1,212 -1,0 +1,199 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.util.shell.command;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.util.Iterator;
+import java.util.Map.Entry;
+
- import org.junit.Assert;
-
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.mock.MockShell;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.format.Formatter;
+import org.apache.accumulo.core.util.shell.Shell;
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
++import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Uses the MockShell to test the shell output with Formatters
+ */
+public class FormatterCommandTest {
+ Writer writer = null;
+ InputStream in = null;
+
+ @Test
+ public void test() throws IOException, AccumuloException, AccumuloSecurityException, TableExistsException, ClassNotFoundException {
+ // Keep the Shell AUDIT log off the test output
+ Logger.getLogger(Shell.class).setLevel(Level.WARN);
+
+ final String[] args = new String[] {"--fake", "-u", "root", "-p", ""};
+
+ final String[] commands = createCommands();
+
+ in = MockShell.makeCommands(commands);
+ writer = new StringWriter();
+
+ final MockShell shell = new MockShell(in, writer);
+ shell.config(args);
+
+ // Can't call createtable in the shell with MockAccumulo
+ shell.getConnector().tableOperations().create("test");
+
+ try {
+ shell.start();
+ } catch (Exception e) {
+ Assert.fail("Exception while running commands: " + e.getMessage());
+ }
+
+ shell.getReader().flushConsole();
+
+ final String[] output = StringUtils.split(writer.toString(), '\n');
+
+ boolean formatterOn = false;
+
+ final String[] expectedDefault = new String[] {
+ "row cf:cq [] 1234abcd",
+ "row cf1:cq1 [] 9876fedc",
+ "row2 cf:cq [] 13579bdf",
+ "row2 cf1:cq [] 2468ace"
+ };
+
+ final String[] expectedFormatted = new String[] {
+ "row cf:cq [] 0x31 0x32 0x33 0x34 0x61 0x62 0x63 0x64",
+ "row cf1:cq1 [] 0x39 0x38 0x37 0x36 0x66 0x65 0x64 0x63",
+ "row2 cf:cq [] 0x31 0x33 0x35 0x37 0x39 0x62 0x64 0x66",
+ "row2 cf1:cq [] 0x32 0x34 0x36 0x38 0x61 0x63 0x65"
+ };
+
+ int outputIndex = 0;
+ while (outputIndex < output.length) {
+ final String line = output[outputIndex];
+
+ if (line.startsWith("root@mock-instance")) {
+ if (line.contains("formatter -t test -f org.apache.accumulo.core.util.shell.command.FormatterCommandTest$HexFormatter")) {
+ formatterOn = true;
+ }
+
+ outputIndex++;
+ } else if (line.startsWith("row")) {
+ int expectedIndex = 0;
+ String[] comparisonData;
+
+ // Pick the type of data we expect (formatted or default)
+ if (formatterOn) {
+ comparisonData = expectedFormatted;
+ } else {
+ comparisonData = expectedDefault;
+ }
+
+ // Ensure each output is what we expected
+ while (expectedIndex + outputIndex < output.length && expectedIndex < expectedFormatted.length) {
+ Assert.assertEquals(comparisonData[expectedIndex].trim(), output[expectedIndex + outputIndex].trim());
+ expectedIndex++;
+ }
+
+ outputIndex += expectedIndex;
+ }
+ }
+ }
+
+ private String[] createCommands() {
+ return new String[] {
+ "table test",
+ "insert row cf cq 1234abcd",
+ "insert row cf1 cq1 9876fedc",
+ "insert row2 cf cq 13579bdf",
+ "insert row2 cf1 cq 2468ace",
+ "scan",
+ "formatter -t test -f org.apache.accumulo.core.util.shell.command.FormatterCommandTest$HexFormatter",
+ "scan"
+ };
+ }
+
+ /**
+ * <p>Simple <code>Formatter</code> that will convert each character in the Value
+ * from decimal to hexadecimal. Will automatically skip over characters in the value
+ * which do not fall within the [0-9,a-f] range.</p>
+ *
+ * <p>Example: <code>'0'</code> will be displayed as <code>'0x30'</code></p>
+ */
+ public static class HexFormatter implements Formatter {
+ private Iterator<Entry<Key, Value>> iter = null;
+ private boolean printTs = false;
+
+ private final static String tab = "\t";
+ private final static String newline = "\n";
+
+ public HexFormatter() {}
+
- /* (non-Javadoc)
- * @see java.util.Iterator#hasNext()
- */
+ @Override
+ public boolean hasNext() {
+ return this.iter.hasNext();
+ }
+
- /* (non-Javadoc)
- * @see java.util.Iterator#next()
- */
+ @Override
+ public String next() {
+ final Entry<Key, Value> entry = iter.next();
+
+ String key;
+
+ // Observe the timestamps
+ if (printTs) {
+ key = entry.getKey().toString();
+ } else {
+ key = entry.getKey().toStringNoTime();
+ }
+
+ final Value v = entry.getValue();
+
+ // Approximate how much space we'll need
+ final StringBuilder sb = new StringBuilder(key.length() + v.getSize() * 5);
+
+ sb.append(key).append(tab);
+
+ for (byte b : v.get()) {
+ if ((b >= 48 && b <= 57) || (b >= 97 || b <= 102)) {
+ sb.append(String.format("0x%x ", Integer.valueOf(b)));
+ }
+ }
+
+ sb.append(newline);
+
+ return sb.toString();
+ }
+
- /* (non-Javadoc)
- * @see java.util.Iterator#remove()
- */
+ @Override
+ public void remove() {
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.util.format.Formatter#initialize(java.lang.Iterable, boolean)
- */
+ @Override
+ public void initialize(final Iterable<Entry<Key,Value>> scanner, final boolean printTimestamps) {
+ this.iter = scanner.iterator();
+ this.printTs = printTimestamps;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
index 2ae82b4,0000000..a3bcf62
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
@@@ -1,234 -1,0 +1,229 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.client;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+import java.util.Random;
+
+import org.apache.accumulo.core.cli.BatchScannerOpts;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Internal class used to verify validity of data read.
+ */
+class CountingVerifyingReceiver {
+ private static final Logger log = Logger.getLogger(CountingVerifyingReceiver.class);
+
+ long count = 0;
+ int expectedValueSize = 0;
+ HashMap<Text,Boolean> expectedRows;
+
+ CountingVerifyingReceiver(HashMap<Text,Boolean> expectedRows, int expectedValueSize) {
+ this.expectedRows = expectedRows;
+ this.expectedValueSize = expectedValueSize;
+ }
+
+ public void receive(Key key, Value value) {
+
+ String row = key.getRow().toString();
+ long rowid = Integer.parseInt(row.split("_")[1]);
+
+ byte expectedValue[] = RandomBatchWriter.createValue(rowid, expectedValueSize);
+
+ if (!Arrays.equals(expectedValue, value.get())) {
+ log.error("Got unexpected value for " + key + " expected : " + new String(expectedValue) + " got : " + new String(value.get()));
+ }
+
+ if (!expectedRows.containsKey(key.getRow())) {
+ log.error("Got unexpected key " + key);
+ } else {
+ expectedRows.put(key.getRow(), true);
+ }
+
+ count++;
+ }
+}
+
+/**
+ * Simple example for reading random batches of data from Accumulo. See docs/examples/README.batch for instructions.
+ */
+public class RandomBatchScanner {
+ private static final Logger log = Logger.getLogger(CountingVerifyingReceiver.class);
+
+ /**
+ * Generate a number of ranges, each covering a single random row.
+ *
+ * @param num
+ * the number of ranges to generate
+ * @param min
+ * the minimum row that will be generated
+ * @param max
+ * the maximum row that will be generated
+ * @param r
+ * a random number generator
+ * @param ranges
+ * a set in which to store the generated ranges
+ * @param expectedRows
+ * a map in which to store the rows covered by the ranges (initially mapped to false)
+ */
+ static void generateRandomQueries(int num, long min, long max, Random r, HashSet<Range> ranges, HashMap<Text,Boolean> expectedRows) {
+ log.info(String.format("Generating %,d random queries...", num));
+ while (ranges.size() < num) {
+ long rowid = (Math.abs(r.nextLong()) % (max - min)) + min;
+
+ Text row1 = new Text(String.format("row_%010d", rowid));
+
+ Range range = new Range(new Text(row1));
+ ranges.add(range);
+ expectedRows.put(row1, false);
+ }
+
+ log.info("finished");
+ }
+
+ /**
+ * Prints a count of the number of rows mapped to false.
+ *
+ * @param expectedRows
+ * @return boolean indicating "were all the rows found?"
+ */
+ private static boolean checkAllRowsFound(HashMap<Text,Boolean> expectedRows) {
+ int count = 0;
+ boolean allFound = true;
+ for (Entry<Text,Boolean> entry : expectedRows.entrySet())
+ if (!entry.getValue())
+ count++;
+
+ if (count > 0) {
+ log.warn("Did not find " + count + " rows");
+ allFound = false;
+ }
+ return allFound;
+ }
+
+ /**
+ * Generates a number of random queries, verifies that the key/value pairs returned were in the queried ranges and that the values were generated by
+ * {@link RandomBatchWriter#createValue(long, int)}. Prints information about the results.
+ *
+ * @param num
+ * the number of queries to generate
+ * @param min
+ * the min row to query
+ * @param max
+ * the max row to query
+ * @param evs
+ * the expected size of the values
+ * @param r
+ * a random number generator
+ * @param tsbr
+ * a batch scanner
+ * @return boolean indicating "did the queries go fine?"
+ */
+ static boolean doRandomQueries(int num, long min, long max, int evs, Random r, BatchScanner tsbr) {
+
+ HashSet<Range> ranges = new HashSet<Range>(num);
+ HashMap<Text,Boolean> expectedRows = new java.util.HashMap<Text,Boolean>();
+
+ generateRandomQueries(num, min, max, r, ranges, expectedRows);
+
+ tsbr.setRanges(ranges);
+
+ CountingVerifyingReceiver receiver = new CountingVerifyingReceiver(expectedRows, evs);
+
+ long t1 = System.currentTimeMillis();
+
+ for (Entry<Key,Value> entry : tsbr) {
+ receiver.receive(entry.getKey(), entry.getValue());
+ }
+
+ long t2 = System.currentTimeMillis();
+
+ log.info(String.format("%6.2f lookups/sec %6.2f secs%n", num / ((t2 - t1) / 1000.0), ((t2 - t1) / 1000.0)));
+ log.info(String.format("num results : %,d%n", receiver.count));
+
+ return checkAllRowsFound(expectedRows);
+ }
+
+ public static class Opts extends ClientOnRequiredTable {
+ @Parameter(names="--min", description="miniumum row that will be generated")
+ long min = 0;
+ @Parameter(names="--max", description="maximum ow that will be generated")
+ long max = 0;
+ @Parameter(names="--num", required=true, description="number of ranges to generate")
+ int num = 0;
+ @Parameter(names="--size", required=true, description="size of the value to write")
+ int size = 0;
+ @Parameter(names="--seed", description="seed for pseudo-random number generator")
+ Long seed = null;
+ }
+
+ /**
+ * Scans over a specified number of entries to Accumulo using a {@link BatchScanner}. Completes scans twice to compare times for a fresh query with those for
+ * a repeated query which has cached metadata and connections already established.
- *
- * @param args
- * @throws AccumuloException
- * @throws AccumuloSecurityException
- * @throws TableNotFoundException
+ */
+ public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ Opts opts = new Opts();
+ BatchScannerOpts bsOpts = new BatchScannerOpts();
+ opts.parseArgs(RandomBatchScanner.class.getName(), args, bsOpts);
+
+ Connector connector = opts.getConnector();
+ BatchScanner batchReader = connector.createBatchScanner(opts.tableName, opts.auths, bsOpts.scanThreads);
+ batchReader.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);
+
+ Random r;
+ if (opts.seed == null)
+ r = new Random();
+ else
+ r = new Random(opts.seed);
+
+ // do one cold
+ boolean status = doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader);
+
+ System.gc();
+ System.gc();
+ System.gc();
+
+ if (opts.seed == null)
+ r = new Random();
+ else
+ r = new Random(opts.seed);
+
+ // do one hot (connections already established, metadata table cached)
+ status = status && doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader);
+
+ batchReader.close();
+ if (!status) {
+ System.exit(1);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
index ce91da6,0000000..e76352a
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
@@@ -1,172 -1,0 +1,168 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.client;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.security.SecurityErrorCode;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Simple example for writing random data to Accumulo. See docs/examples/README.batch for instructions.
+ *
+ * The rows of the entries will be randomly generated numbers between a specified min and max (prefixed by "row_"). The column families will be "foo" and column
+ * qualifiers will be "1". The values will be random byte arrays of a specified size.
+ */
+public class RandomBatchWriter {
+
+ /**
+ * Creates a random byte array of specified size using the specified seed.
+ *
+ * @param rowid
+ * the seed to use for the random number generator
+ * @param dataSize
+ * the size of the array
+ * @return a random byte array
+ */
+ public static byte[] createValue(long rowid, int dataSize) {
+ Random r = new Random(rowid);
+ byte value[] = new byte[dataSize];
+
+ r.nextBytes(value);
+
+ // transform to printable chars
+ for (int j = 0; j < value.length; j++) {
+ value[j] = (byte) (((0xff & value[j]) % 92) + ' ');
+ }
+
+ return value;
+ }
+
+ /**
+ * Creates a mutation on a specified row with column family "foo", column qualifier "1", specified visibility, and a random value of specified size.
+ *
+ * @param rowid
+ * the row of the mutation
+ * @param dataSize
+ * the size of the random value
+ * @param visibility
+ * the visibility of the entry to insert
+ * @return a mutation
+ */
+ public static Mutation createMutation(long rowid, int dataSize, ColumnVisibility visibility) {
+ Text row = new Text(String.format("row_%010d", rowid));
+
+ Mutation m = new Mutation(row);
+
+ // create a random value that is a function of the
+ // row id for verification purposes
+ byte value[] = createValue(rowid, dataSize);
+
+ m.put(new Text("foo"), new Text("1"), visibility, new Value(value));
+
+ return m;
+ }
+
+ static class Opts extends ClientOnRequiredTable {
+ @Parameter(names="--num", required=true)
+ int num = 0;
+ @Parameter(names="--min")
+ long min = 0;
+ @Parameter(names="--max")
+ long max = Long.MAX_VALUE;
+ @Parameter(names="--size", required=true, description="size of the value to write")
+ int size = 0;
+ @Parameter(names="--vis", converter=VisibilityConverter.class)
+ ColumnVisibility visiblity = new ColumnVisibility("");
+ @Parameter(names="--seed", description="seed for pseudo-random number generator")
+ Long seed = null;
+ }
+
+ /**
+ * Writes a specified number of entries to Accumulo using a {@link BatchWriter}.
- *
- * @throws AccumuloException
- * @throws AccumuloSecurityException
- * @throws TableNotFoundException
+ */
+ public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ Opts opts = new Opts();
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(RandomBatchWriter.class.getName(), args, bwOpts);
+ if ((opts.max - opts.min) < opts.num) {
+ System.err.println(String.format("You must specify a min and a max that allow for at least num possible values. For example, you requested %d rows, but a min of %d and a max of %d only allows for %d rows.", opts.num, opts.min, opts.max, (opts.max - opts.min)));
+ System.exit(1);
+ }
+ Random r;
+ if (opts.seed == null)
+ r = new Random();
+ else {
+ r = new Random(opts.seed);
+ }
+ Connector connector = opts.getConnector();
+ BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
+
+ // reuse the ColumnVisibility object to improve performance
+ ColumnVisibility cv = opts.visiblity;
+
+ // Generate num unique row ids in the given range
+ HashSet<Long> rowids = new HashSet<Long>(opts.num);
+ while (rowids.size() < opts.num) {
+ rowids.add((Math.abs(r.nextLong()) % (opts.max - opts.min)) + opts.min);
+ }
+ for (long rowid : rowids) {
+ Mutation m = createMutation(rowid, opts.size, cv);
+ bw.addMutation(m);
+ }
+
+ try {
+ bw.close();
+ } catch (MutationsRejectedException e) {
+ if (e.getAuthorizationFailuresMap().size() > 0) {
+ HashMap<String,Set<SecurityErrorCode>> tables = new HashMap<String,Set<SecurityErrorCode>>();
+ for (Entry<KeyExtent,Set<SecurityErrorCode>> ke : e.getAuthorizationFailuresMap().entrySet()) {
+ Set<SecurityErrorCode> secCodes = tables.get(ke.getKey().getTableId().toString());
+ if (secCodes == null) {
+ secCodes = new HashSet<SecurityErrorCode>();
+ tables.put(ke.getKey().getTableId().toString(), secCodes);
+ }
+ secCodes.addAll(ke.getValue());
+ }
+ System.err.println("ERROR : Not authorized to write to tables : " + tables);
+ }
+
+ if (e.getConstraintViolationSummaries().size() > 0) {
+ System.err.println("ERROR : Constraint violations occurred : " + e.getConstraintViolationSummaries());
+ }
+ System.exit(1);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java
index a56fdc0,0000000..c37c1c3
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java
@@@ -1,73 -1,0 +1,68 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.client;
+
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.security.ColumnVisibility;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Simple example for writing random data in sequential order to Accumulo. See docs/examples/README.batch for instructions.
+ */
+public class SequentialBatchWriter {
+
+ static class Opts extends ClientOnRequiredTable {
+ @Parameter(names="--start")
+ long start = 0;
+ @Parameter(names="--num", required=true)
+ long num = 0;
+ @Parameter(names="--size", required=true, description="size of the value to write")
+ int valueSize = 0;
+ @Parameter(names="--vis", converter=VisibilityConverter.class)
+ ColumnVisibility vis = new ColumnVisibility();
+ }
+
+ /**
+ * Writes a specified number of entries to Accumulo using a {@link BatchWriter}. The rows of the entries will be sequential starting at a specified number.
+ * The column families will be "foo" and column qualifiers will be "1". The values will be random byte arrays of a specified size.
- *
- * @throws AccumuloException
- * @throws AccumuloSecurityException
- * @throws TableNotFoundException
- * @throws MutationsRejectedException
+ */
+ public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, MutationsRejectedException {
+ Opts opts = new Opts();
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(SequentialBatchWriter.class.getName(), args, bwOpts);
+ Connector connector = opts.getConnector();
+ BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
+
+ long end = opts.start + opts.num;
+
+ for (long i = opts.start; i < end; i++) {
+ Mutation m = RandomBatchWriter.createMutation(i, opts.valueSize, opts.vis);
+ bw.addMutation(m);
+ }
+
+ bw.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/TraceDumpExample.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/TraceDumpExample.java
index 2947e0e,0000000..70a23e5
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/TraceDumpExample.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/TraceDumpExample.java
@@@ -1,77 -1,0 +1,72 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.client;
+
+import org.apache.accumulo.core.cli.ClientOnDefaultTable;
+import org.apache.accumulo.core.cli.ScannerOpts;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.trace.TraceDump;
+import org.apache.accumulo.core.trace.TraceDump.Printer;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Example of using the TraceDump class to print a formatted view of a Trace
+ *
+ */
+public class TraceDumpExample {
+
+ static class Opts extends ClientOnDefaultTable {
+ public Opts() {
+ super("trace");
+ }
+
+ @Parameter(names = {"--traceid"}, description = "The hex string id of a given trace, for example 16cfbbd7beec4ae3")
+ public String traceId = "";
+ }
+
+ public void dump(Opts opts) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+
+ if (opts.traceId.isEmpty()) {
+ throw new IllegalArgumentException("--traceid option is required");
+ }
+
+ Scanner scanner = opts.getConnector().createScanner(opts.getTableName(), opts.auths);
+ scanner.setRange(new Range(new Text(opts.traceId)));
+ TraceDump.printTrace(scanner, new Printer() {
- public void print(String line) {
++ @Override
++ public void print(String line) {
+ System.out.println(line);
+ }
+ });
+ }
+
- /**
- * @param args
- * @throws AccumuloSecurityException
- * @throws AccumuloException
- * @throws TableNotFoundException
- */
+ public static void main(String[] args) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ TraceDumpExample traceDumpExample = new TraceDumpExample();
+ Opts opts = new Opts();
+ ScannerOpts scannerOpts = new ScannerOpts();
+ opts.parseArgs(TraceDumpExample.class.getName(), args, scannerOpts);
+
+ traceDumpExample.dump(opts);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java
index 744efed,0000000..f11c739
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java
@@@ -1,283 -1,0 +1,280 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.dirlist;
+
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Provides utility methods for getting the info for a file, listing the contents of a directory, and performing single wild card searches on file or directory
+ * names. See docs/examples/README.dirlist for instructions.
+ */
+public class QueryUtil {
+ private Connector conn = null;
+ private String tableName;
+ private Authorizations auths;
+ public static final Text DIR_COLF = new Text("dir");
+ public static final Text FORWARD_PREFIX = new Text("f");
+ public static final Text REVERSE_PREFIX = new Text("r");
+ public static final Text INDEX_COLF = new Text("i");
+ public static final Text COUNTS_COLQ = new Text("counts");
+
+ public QueryUtil(Opts opts) throws AccumuloException,
+ AccumuloSecurityException {
+ conn = opts.getConnector();
+ this.tableName = opts.tableName;
+ this.auths = opts.auths;
+ }
+
+ /**
+ * Calculates the depth of a path, i.e. the number of forward slashes in the path name.
+ *
+ * @param path
+ * the full path of a file or directory
+ * @return the depth of the path
+ */
+ public static int getDepth(String path) {
+ int numSlashes = 0;
+ int index = -1;
+ while ((index = path.indexOf("/", index + 1)) >= 0)
+ numSlashes++;
+ return numSlashes;
+ }
+
+ /**
+ * Given a path, construct an accumulo row prepended with the path's depth for the directory table.
+ *
+ * @param path
+ * the full path of a file or directory
+ * @return the accumulo row associated with this path
+ */
+ public static Text getRow(String path) {
+ Text row = new Text(String.format("%03d", getDepth(path)));
+ row.append(path.getBytes(), 0, path.length());
+ return row;
+ }
+
+ /**
+ * Given a path, construct an accumulo row prepended with the {@link #FORWARD_PREFIX} for the index table.
+ *
+ * @param path
+ * the full path of a file or directory
+ * @return the accumulo row associated with this path
+ */
+ public static Text getForwardIndex(String path) {
+ String part = path.substring(path.lastIndexOf("/") + 1);
+ if (part.length() == 0)
+ return null;
+ Text row = new Text(FORWARD_PREFIX);
+ row.append(part.getBytes(), 0, part.length());
+ return row;
+ }
+
+ /**
+ * Given a path, construct an accumulo row prepended with the {@link #REVERSE_PREFIX} with the path reversed for the index table.
+ *
+ * @param path
+ * the full path of a file or directory
+ * @return the accumulo row associated with this path
+ */
+ public static Text getReverseIndex(String path) {
+ String part = path.substring(path.lastIndexOf("/") + 1);
+ if (part.length() == 0)
+ return null;
+ byte[] rev = new byte[part.length()];
+ int i = part.length() - 1;
+ for (byte b : part.getBytes())
+ rev[i--] = b;
+ Text row = new Text(REVERSE_PREFIX);
+ row.append(rev, 0, rev.length);
+ return row;
+ }
+
+ /**
+ * Returns either the {@link #DIR_COLF} or a decoded string version of the colf.
+ *
+ * @param colf
+ * the column family
+ */
+ public static String getType(Text colf) {
+ if (colf.equals(DIR_COLF))
+ return colf.toString() + ":";
+ return Long.toString(Ingest.encoder.decode(colf.getBytes())) + ":";
+ }
+
+ /**
+ * Scans over the directory table and pulls out stat information about a path.
+ *
+ * @param path
+ * the full path of a file or directory
+ */
+ public Map<String,String> getData(String path) throws TableNotFoundException {
+ if (path.endsWith("/"))
+ path = path.substring(0, path.length() - 1);
+ Scanner scanner = conn.createScanner(tableName, auths);
+ scanner.setRange(new Range(getRow(path)));
+ Map<String,String> data = new TreeMap<String,String>();
+ for (Entry<Key,Value> e : scanner) {
+ String type = getType(e.getKey().getColumnFamily());
+ data.put("fullname", e.getKey().getRow().toString().substring(3));
+ data.put(type + e.getKey().getColumnQualifier().toString() + ":" + e.getKey().getColumnVisibility().toString(), new String(e.getValue().get()));
+ }
+ return data;
+ }
+
+ /**
+ * Uses the directory table to list the contents of a directory.
+ *
+ * @param path
+ * the full path of a directory
+ */
+ public Map<String,Map<String,String>> getDirList(String path) throws TableNotFoundException {
+ if (!path.endsWith("/"))
+ path = path + "/";
+ Map<String,Map<String,String>> fim = new TreeMap<String,Map<String,String>>();
+ Scanner scanner = conn.createScanner(tableName, auths);
+ scanner.setRange(Range.prefix(getRow(path)));
+ for (Entry<Key,Value> e : scanner) {
+ String name = e.getKey().getRow().toString();
+ name = name.substring(name.lastIndexOf("/") + 1);
+ String type = getType(e.getKey().getColumnFamily());
+ if (!fim.containsKey(name)) {
+ fim.put(name, new TreeMap<String,String>());
+ fim.get(name).put("fullname", e.getKey().getRow().toString().substring(3));
+ }
+ fim.get(name).put(type + e.getKey().getColumnQualifier().toString() + ":" + e.getKey().getColumnVisibility().toString(), new String(e.getValue().get()));
+ }
+ return fim;
+ }
+
+ /**
+ * Scans over the index table for files or directories with a given name.
+ *
+ * @param term
+ * the name a file or directory to search for
+ */
+ public Iterable<Entry<Key,Value>> exactTermSearch(String term) throws Exception {
+ System.out.println("executing exactTermSearch for " + term);
+ Scanner scanner = conn.createScanner(tableName, auths);
+ scanner.setRange(new Range(getForwardIndex(term)));
+ return scanner;
+ }
+
+ /**
+ * Scans over the index table for files or directories with a given name, prefix, or suffix (indicated by a wildcard '*' at the beginning or end of the term.
+ *
+ * @param exp
+ * the name a file or directory to search for with an optional wildcard '*' at the beginning or end
+ */
+ public Iterable<Entry<Key,Value>> singleRestrictedWildCardSearch(String exp) throws Exception {
+ if (exp.indexOf("/") >= 0)
+ throw new Exception("this method only works with unqualified names");
+
+ Scanner scanner = conn.createScanner(tableName, auths);
+ if (exp.startsWith("*")) {
+ System.out.println("executing beginning wildcard search for " + exp);
+ exp = exp.substring(1);
+ scanner.setRange(Range.prefix(getReverseIndex(exp)));
+ } else if (exp.endsWith("*")) {
+ System.out.println("executing ending wildcard search for " + exp);
+ exp = exp.substring(0, exp.length() - 1);
+ scanner.setRange(Range.prefix(getForwardIndex(exp)));
+ } else if (exp.indexOf("*") >= 0) {
+ throw new Exception("this method only works for beginning or ending wild cards");
+ } else {
+ return exactTermSearch(exp);
+ }
+ return scanner;
+ }
+
+ /**
+ * Scans over the index table for files or directories with a given name that can contain a single wildcard '*' anywhere in the term.
+ *
+ * @param exp
+ * the name a file or directory to search for with one optional wildcard '*'
+ */
+ public Iterable<Entry<Key,Value>> singleWildCardSearch(String exp) throws Exception {
+ int starIndex = exp.indexOf("*");
+ if (exp.indexOf("*", starIndex + 1) >= 0)
+ throw new Exception("only one wild card for search");
+
+ if (starIndex < 0) {
+ return exactTermSearch(exp);
+ } else if (starIndex == 0 || starIndex == exp.length() - 1) {
+ return singleRestrictedWildCardSearch(exp);
+ }
+
+ String firstPart = exp.substring(0, starIndex);
+ String lastPart = exp.substring(starIndex + 1);
+ String regexString = ".*/" + exp.replace("*", "[^/]*");
+
+ Scanner scanner = conn.createScanner(tableName, auths);
+ if (firstPart.length() >= lastPart.length()) {
+ System.out.println("executing middle wildcard search for " + regexString + " from entries starting with " + firstPart);
+ scanner.setRange(Range.prefix(getForwardIndex(firstPart)));
+ } else {
+ System.out.println("executing middle wildcard search for " + regexString + " from entries ending with " + lastPart);
+ scanner.setRange(Range.prefix(getReverseIndex(lastPart)));
+ }
+ IteratorSetting regex = new IteratorSetting(50, "regex", RegExFilter.class);
+ RegExFilter.setRegexs(regex, null, null, regexString, null, false);
+ scanner.addScanIterator(regex);
+ return scanner;
+ }
+
+ public static class Opts extends ClientOnRequiredTable {
+ @Parameter(names="--path", description="the directory to list")
+ String path = "/";
+ @Parameter(names="--search", description="find a file or directorys with the given name")
+ boolean search = false;
+ }
+
+ /**
+ * Lists the contents of a directory using the directory table, or searches for file or directory names (if the -search flag is included).
- *
- * @param args
- * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(QueryUtil.class.getName(), args);
+ QueryUtil q = new QueryUtil(opts);
+ if (opts.search) {
+ for (Entry<Key,Value> e : q.singleWildCardSearch(opts.path)) {
+ System.out.println(e.getKey().getColumnQualifier());
+ }
+ } else {
+ for (Entry<String,Map<String,String>> e : q.getDirList(opts.path).entrySet()) {
+ System.out.println(e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/NGramIngest.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/NGramIngest.java
index cd6ca40,0000000..dc14512
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/NGramIngest.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/NGramIngest.java
@@@ -1,116 -1,0 +1,113 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.mapreduce;
+
+import java.io.IOException;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Map job to ingest n-gram files from
+ * http://storage.googleapis.com/books/ngrams/books/datasetsv2.html
+ */
+public class NGramIngest extends Configured implements Tool {
+
+ private static final Logger log = Logger.getLogger(NGramIngest.class);
+
+
+ static class Opts extends ClientOnRequiredTable {
+ @Parameter(names = "--input", required=true)
+ String inputDirectory;
+ }
+ static class NGramMapper extends Mapper<LongWritable, Text, Text, Mutation> {
+
+ @Override
+ protected void map(LongWritable location, Text value, Context context) throws IOException, InterruptedException {
+ String parts[] = value.toString().split("\\t");
+ if (parts.length >= 4) {
+ Mutation m = new Mutation(parts[0]);
+ m.put(parts[1], String.format("%010d", Long.parseLong(parts[2])), new Value(parts[3].trim().getBytes()));
+ context.write(null, m);
+ }
+ }
+ }
+
- /**
- * @param args
- */
+ @Override
+ public int run(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(getClass().getName(), args);
+
+ Job job = new Job(getConf(), getClass().getSimpleName());
+ job.setJarByClass(getClass());
+
+ opts.setAccumuloConfigs(job);
+ job.setInputFormatClass(TextInputFormat.class);
+ job.setOutputFormatClass(AccumuloOutputFormat.class);
+
+ job.setMapperClass(NGramMapper.class);
+ job.setMapOutputKeyClass(Text.class);
+ job.setMapOutputValueClass(Mutation.class);
+
+ job.setNumReduceTasks(0);
+ job.setSpeculativeExecution(false);
+
+
+ if (!opts.getConnector().tableOperations().exists(opts.tableName)) {
+ log.info("Creating table " + opts.tableName);
+ opts.getConnector().tableOperations().create(opts.tableName);
+ SortedSet<Text> splits = new TreeSet<Text>();
+ String numbers[] = "1 2 3 4 5 6 7 8 9".split("\\s");
+ String lower[] = "a b c d e f g h i j k l m n o p q r s t u v w x y z".split("\\s");
+ String upper[] = "A B C D E F G H I J K L M N O P Q R S T U V W X Y Z".split("\\s");
+ for (String[] array : new String[][]{numbers, lower, upper}) {
+ for (String s : array) {
+ splits.add(new Text(s));
+ }
+ }
+ opts.getConnector().tableOperations().addSplits(opts.tableName, splits);
+ }
+
+ TextInputFormat.addInputPath(job, new Path(opts.inputDirectory));
+ job.waitForCompletion(true);
+ return job.isSuccessful() ? 0 : 1;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(CachedConfiguration.getInstance(), new NGramIngest(), args);
+ if (res != 0)
+ System.exit(res);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
index d8eedef,0000000..669c76d
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
@@@ -1,130 -1,0 +1,129 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.mapreduce;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Map;
+
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.format.DefaultFormatter;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Takes a table and outputs the specified column to a set of part files on hdfs accumulo accumulo.examples.mapreduce.TableToFile <username> <password>
+ * <tablename> <column> <hdfs-output-path>
+ */
+public class TableToFile extends Configured implements Tool {
+
+ static class Opts extends ClientOnRequiredTable {
+ @Parameter(names = "--output", description = "output directory", required = true)
+ String output;
+ @Parameter(names = "--columns", description = "columns to extract, in cf:cq{,cf:cq,...} form")
+ String columns = "";
+ }
+
+ /**
+ * The Mapper class that given a row number, will generate the appropriate output line.
+ */
+ public static class TTFMapper extends Mapper<Key,Value,NullWritable,Text> {
+ @Override
+ public void map(Key row, Value data, Context context) throws IOException, InterruptedException {
+ final Key r = row;
+ final Value v = data;
+ Map.Entry<Key,Value> entry = new Map.Entry<Key,Value>() {
+ @Override
+ public Key getKey() {
+ return r;
+ }
+
+ @Override
+ public Value getValue() {
+ return v;
+ }
+
+ @Override
+ public Value setValue(Value value) {
+ return null;
+ }
+ };
+ context.write(NullWritable.get(), new Text(DefaultFormatter.formatEntry(entry, false)));
+ context.setStatus("Outputed Value");
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException, AccumuloSecurityException {
+ Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+ job.setJarByClass(this.getClass());
+ Opts opts = new Opts();
+ opts.parseArgs(getClass().getName(), args);
+
+ job.setInputFormatClass(AccumuloInputFormat.class);
+ opts.setAccumuloConfigs(job);
+
+ HashSet<Pair<Text,Text>> columnsToFetch = new HashSet<Pair<Text,Text>>();
+ for (String col : opts.columns.split(",")) {
+ int idx = col.indexOf(":");
+ Text cf = new Text(idx < 0 ? col : col.substring(0, idx));
+ Text cq = idx < 0 ? null : new Text(col.substring(idx + 1));
+ if (cf.getLength() > 0)
+ columnsToFetch.add(new Pair<Text,Text>(cf, cq));
+ }
+ if (!columnsToFetch.isEmpty())
+ AccumuloInputFormat.fetchColumns(job, columnsToFetch);
+
+ job.setMapperClass(TTFMapper.class);
+ job.setMapOutputKeyClass(NullWritable.class);
+ job.setMapOutputValueClass(Text.class);
+
+ job.setNumReduceTasks(0);
+
+ job.setOutputFormatClass(TextOutputFormat.class);
+ TextOutputFormat.setOutputPath(job, new Path(opts.output));
+
+ job.waitForCompletion(true);
+ return job.isSuccessful() ? 0 : 1;
+ }
+
+ /**
+ *
+ * @param args
+ * instanceName zookeepers username password table columns outputpath
- * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(CachedConfiguration.getInstance(), new TableToFile(), args);
+ if (res != 0)
+ System.exit(res);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
index f6d610e,0000000..d98d78b
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
@@@ -1,78 -1,0 +1,75 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.shard;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.cli.BatchScannerOpts;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.IntersectingIterator;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * This program queries a set of terms in the shard table (populated by {@link Index}) using the {@link IntersectingIterator}.
+ *
+ * See docs/examples/README.shard for instructions.
+ */
+
+public class Query {
+
+ static class Opts extends ClientOnRequiredTable {
+ @Parameter(description=" term { <term> ... }")
+ List<String> terms = new ArrayList<String>();
+ }
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ BatchScannerOpts bsOpts = new BatchScannerOpts();
+ opts.parseArgs(Query.class.getName(), args, bsOpts);
+
+ Connector conn = opts.getConnector();
+ BatchScanner bs = conn.createBatchScanner(opts.tableName, opts.auths, bsOpts.scanThreads);
+ bs.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);
+
+ Text columns[] = new Text[opts.terms.size()];
+ int i = 0;
+ for (String term : opts.terms) {
+ columns[i++] = new Text(term);
+ }
+ IteratorSetting ii = new IteratorSetting(20, "ii", IntersectingIterator.class);
+ IntersectingIterator.setColumnFamilies(ii, columns);
+ bs.addScanIterator(ii);
+ bs.setRanges(Collections.singleton(new Range()));
+ for (Entry<Key,Value> entry : bs) {
+ System.out.println(" " + entry.getKey().getColumnQualifier());
+ }
+
+ }
+
+}
[05/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
index 36bd86a,0000000..8873938
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
+++ b/server/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
@@@ -1,157 -1,0 +1,156 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security.handler;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+public class ZKAuthorizor implements Authorizor {
+ private static final Logger log = Logger.getLogger(ZKAuthorizor.class);
+ private static Authorizor zkAuthorizorInstance = null;
+
+ private final String ZKUserAuths = "/Authorizations";
+
+ private String ZKUserPath;
+ private final ZooCache zooCache;
+
+ public static synchronized Authorizor getInstance() {
+ if (zkAuthorizorInstance == null)
+ zkAuthorizorInstance = new ZKAuthorizor();
+ return zkAuthorizorInstance;
+ }
+
+ public ZKAuthorizor() {
+ zooCache = new ZooCache();
+ }
+
++ @Override
+ public void initialize(String instanceId, boolean initialize) {
+ ZKUserPath = ZKSecurityTool.getInstancePath(instanceId) + "/users";
+ }
+
++ @Override
+ public Authorizations getCachedUserAuthorizations(String user) {
+ byte[] authsBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserAuths);
+ if (authsBytes != null)
+ return ZKSecurityTool.convertAuthorizations(authsBytes);
+ return Constants.NO_AUTHS;
+ }
+
+ @Override
+ public boolean validSecurityHandlers(Authenticator auth, PermissionHandler pm) {
+ return true;
+ }
+
+ @Override
+ public void initializeSecurity(TCredentials itw, String rootuser) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+
+ // create the root user with all system privileges, no table privileges, and no record-level authorizations
+ Set<SystemPermission> rootPerms = new TreeSet<SystemPermission>();
+ for (SystemPermission p : SystemPermission.values())
+ rootPerms.add(p);
+ Map<String,Set<TablePermission>> tablePerms = new HashMap<String,Set<TablePermission>>();
+ // Allow the root user to flush the !METADATA table
+ tablePerms.put(Constants.METADATA_TABLE_ID, Collections.singleton(TablePermission.ALTER_TABLE));
+
+ try {
+ // prep parent node of users with root username
+ if (!zoo.exists(ZKUserPath))
+ zoo.putPersistentData(ZKUserPath, rootuser.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
+
+ initUser(rootuser);
+ zoo.putPersistentData(ZKUserPath + "/" + rootuser + ZKUserAuths, ZKSecurityTool.convertAuthorizations(Constants.NO_AUTHS), NodeExistsPolicy.FAIL);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
- /**
- * @param user
- * @throws AccumuloSecurityException
- */
++ @Override
+ public void initUser(String user) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ try {
+ zoo.putPersistentData(ZKUserPath + "/" + user, new byte[0], NodeExistsPolicy.SKIP);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void dropUser(String user) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserAuths, NodeMissingPolicy.SKIP);
+ zooCache.clear(ZKUserPath + "/" + user);
+ }
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ if (e.code().equals(KeeperException.Code.NONODE))
+ throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+
+ }
+ }
+
+ @Override
+ public void changeAuthorizations(String user, Authorizations authorizations) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserAuths, ZKSecurityTool.convertAuthorizations(authorizations),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
index 4a05658,0000000..d802eb9
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
+++ b/server/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
@@@ -1,365 -1,0 +1,363 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security.handler;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.Code;
+
+/**
+ *
+ */
+public class ZKPermHandler implements PermissionHandler {
+ private static final Logger log = Logger.getLogger(ZKAuthorizor.class);
+ private static PermissionHandler zkPermHandlerInstance = null;
+
+ private String ZKUserPath;
+ private String ZKTablePath;
+ private final ZooCache zooCache;
+ private static final String ZKUserSysPerms = "/System";
+ private static final String ZKUserTablePerms = "/Tables";
+
+ public static synchronized PermissionHandler getInstance() {
+ if (zkPermHandlerInstance == null)
+ zkPermHandlerInstance = new ZKPermHandler();
+ return zkPermHandlerInstance;
+ }
+
++ @Override
+ public void initialize(String instanceId, boolean initialize) {
+ ZKUserPath = ZKSecurityTool.getInstancePath(instanceId) + "/users";
+ ZKTablePath = ZKSecurityTool.getInstancePath(instanceId) + "/tables";
+ }
+
+ public ZKPermHandler() {
+ zooCache = new ZooCache();
+ }
+
+ @Override
+ public boolean hasTablePermission(String user, String table, TablePermission permission) throws TableNotFoundException {
+ byte[] serializedPerms;
+ try {
+ String path = ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table;
+ ZooReaderWriter.getRetryingInstance().sync(path);
+ serializedPerms = ZooReaderWriter.getRetryingInstance().getData(path, null);
+ } catch (KeeperException e) {
+ if (e.code() == Code.NONODE) {
+ // maybe the table was just deleted?
+ try {
+ // check for existence:
+ ZooReaderWriter.getRetryingInstance().getData(ZKTablePath + "/" + table, null);
+ // it's there, you don't have permission
+ return false;
+ } catch (InterruptedException ex) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ } catch (KeeperException ex) {
+ // not there, throw an informative exception
+ if (e.code() == Code.NONODE) {
+ throw new TableNotFoundException(null, table, "while checking permissions");
+ }
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ }
+ return false;
+ }
+ log.warn("Unhandled KeeperException, failing closed for table permission check", e);
+ return false;
+ } catch (InterruptedException e) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ }
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertTablePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean hasCachedTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertTablePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public void grantSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ try {
+ byte[] permBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+ Set<SystemPermission> perms;
+ if (permBytes == null) {
+ perms = new TreeSet<SystemPermission>();
+ } else {
+ perms = ZKSecurityTool.convertSystemPermissions(permBytes);
+ }
+
+ if (perms.add(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(perms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void grantTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException {
+ Set<TablePermission> tablePerms;
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ if (serializedPerms != null)
+ tablePerms = ZKSecurityTool.convertTablePermissions(serializedPerms);
+ else
+ tablePerms = new TreeSet<TablePermission>();
+
+ try {
+ if (tablePerms.add(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, ZKSecurityTool.convertTablePermissions(tablePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void revokeSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] sysPermBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+
+ // User had no system permission, nothing to revoke.
+ if (sysPermBytes == null)
+ return;
+
+ Set<SystemPermission> sysPerms = ZKSecurityTool.convertSystemPermissions(sysPermBytes);
+
+ try {
+ if (sysPerms.remove(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(sysPerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void revokeTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+
+ // User had no table permission, nothing to revoke.
+ if (serializedPerms == null)
+ return;
+
+ Set<TablePermission> tablePerms = ZKSecurityTool.convertTablePermissions(serializedPerms);
+ try {
+ if (tablePerms.remove(permission)) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ if (tablePerms.size() == 0)
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, NodeMissingPolicy.SKIP);
+ else
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, ZKSecurityTool.convertTablePermissions(tablePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void cleanTablePermissions(String table) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ for (String user : zooCache.getChildren(ZKUserPath))
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, NodeMissingPolicy.SKIP);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException("unknownUser", SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void initializeSecurity(TCredentials itw, String rootuser) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+
+ // create the root user with all system privileges, no table privileges, and no record-level authorizations
+ Set<SystemPermission> rootPerms = new TreeSet<SystemPermission>();
+ for (SystemPermission p : SystemPermission.values())
+ rootPerms.add(p);
+ Map<String,Set<TablePermission>> tablePerms = new HashMap<String,Set<TablePermission>>();
+ // Allow the root user to flush the !METADATA table
+ tablePerms.put(Constants.METADATA_TABLE_ID, Collections.singleton(TablePermission.ALTER_TABLE));
+
+ try {
+ // prep parent node of users with root username
+ if (!zoo.exists(ZKUserPath))
+ zoo.putPersistentData(ZKUserPath, rootuser.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
+
+ initUser(rootuser);
+ zoo.putPersistentData(ZKUserPath + "/" + rootuser + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(rootPerms), NodeExistsPolicy.FAIL);
+ for (Entry<String,Set<TablePermission>> entry : tablePerms.entrySet())
+ createTablePerm(rootuser, entry.getKey(), entry.getValue());
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
- /**
- * @param user
- * @throws AccumuloSecurityException
- */
++ @Override
+ public void initUser(String user) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ try {
+ zoo.putPersistentData(ZKUserPath + "/" + user, new byte[0], NodeExistsPolicy.SKIP);
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms, new byte[0], NodeExistsPolicy.SKIP);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Sets up a new table configuration for the provided user/table. No checking for existence is done here, it should be done before calling.
+ */
+ private void createTablePerm(String user, String table, Set<TablePermission> perms) throws KeeperException, InterruptedException {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getRetryingInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table,
+ ZKSecurityTool.convertTablePermissions(perms), NodeExistsPolicy.FAIL);
+ }
+ }
+
+ @Override
+ public void cleanUser(String user) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserSysPerms, NodeMissingPolicy.SKIP);
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms, NodeMissingPolicy.SKIP);
+ zooCache.clear(ZKUserPath + "/" + user);
+ }
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ if (e.code().equals(KeeperException.Code.NONODE))
+ throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+
+ }
+ }
+
+ @Override
+ public boolean hasSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] perms;
+ try {
+ String path = ZKUserPath + "/" + user + ZKUserSysPerms;
+ ZooReaderWriter.getRetryingInstance().sync(path);
+ perms = ZooReaderWriter.getRetryingInstance().getData(path, null);
+ } catch (KeeperException e) {
+ if (e.code() == Code.NONODE) {
+ return false;
+ }
+ log.warn("Unhandled KeeperException, failing closed for table permission check", e);
+ return false;
+ } catch (InterruptedException e) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ }
+
+ if (perms == null)
+ return false;
+ return ZKSecurityTool.convertSystemPermissions(perms).contains(permission);
+ }
+
+ @Override
+ public boolean hasCachedSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] perms = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+ if (perms == null)
+ return false;
+ return ZKSecurityTool.convertSystemPermissions(perms).contains(permission);
+ }
+
+ @Override
+ public boolean validSecurityHandlers(Authenticator authent, Authorizor author) {
+ return true;
+ }
+
+ @Override
+ public void initTable(String table) throws AccumuloSecurityException {
+ // All proper housekeeping is done on delete and permission granting, no work needs to be done here
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/tabletserver/MemValue.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/tabletserver/MemValue.java
index 735bf20,0000000..f68859a
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/MemValue.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/MemValue.java
@@@ -1,93 -1,0 +1,95 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.tabletserver;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.accumulo.core.data.Value;
+
+/**
+ *
+ */
+public class MemValue extends Value {
+ int kvCount;
+ boolean merged = false;
+
+ /**
+ * @param value
+ * Value
+ * @param kv
+ * kv count
+ */
+ public MemValue(byte[] value, int kv) {
+ super(value);
+ this.kvCount = kv;
+ }
+
+ public MemValue() {
+ super();
+ this.kvCount = Integer.MAX_VALUE;
+ }
+
+ public MemValue(Value value, int kv) {
+ super(value);
+ this.kvCount = kv;
+ }
+
+ // Override
++ @Override
+ public void write(final DataOutput out) throws IOException {
+ if (!merged) {
+ byte[] combinedBytes = new byte[getSize() + 4];
+ System.arraycopy(value, 0, combinedBytes, 4, getSize());
+ combinedBytes[0] = (byte) (kvCount >>> 24);
+ combinedBytes[1] = (byte) (kvCount >>> 16);
+ combinedBytes[2] = (byte) (kvCount >>> 8);
+ combinedBytes[3] = (byte) (kvCount);
+ value = combinedBytes;
+ merged = true;
+ }
+ super.write(out);
+ }
+
++ @Override
+ public void set(final byte[] b) {
+ super.set(b);
+ merged = false;
+ }
+
++ @Override
+ public void copy(byte[] b) {
+ super.copy(b);
+ merged = false;
+ }
+
+ /**
+ * Takes a Value and will take out the embedded kvCount, and then return that value while replacing the Value with the original unembedded version
+ *
- * @param v
+ * @return The kvCount embedded in v.
+ */
+ public static int splitKVCount(Value v) {
+ if (v instanceof MemValue)
+ return ((MemValue) v).kvCount;
+
+ byte[] originalBytes = new byte[v.getSize() - 4];
+ byte[] combined = v.get();
+ System.arraycopy(combined, 4, originalBytes, 0, originalBytes.length);
+ v.set(originalBytes);
+ return (combined[0] << 24) + ((combined[1] & 0xFF) << 16) + ((combined[2] & 0xFF) << 8) + (combined[3] & 0xFF);
+ }
+}
[20/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
index d018228,0000000..f4a8082
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
@@@ -1,136 -1,0 +1,135 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+public class AddFilesWithMissingEntries {
+
+ static final Logger log = Logger.getLogger(AddFilesWithMissingEntries.class);
+
+ public static class Opts extends ClientOpts {
+ @Parameter(names="-update", description="Make changes to the !METADATA table to include missing files")
+ boolean update = false;
+ }
+
+
+ /**
+ * A utility to add files to the !METADATA table that are not listed in the root tablet.
+ * This is a recovery tool for someone who knows what they are doing. It might be better to
+ * save off files, and recover your instance by re-initializing and importing the existing files.
+ *
- * @param args
+ */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(AddFilesWithMissingEntries.class.getName(), args, bwOpts);
+
+ final Key rootTableEnd = new Key(Constants.ROOT_TABLET_EXTENT.getEndRow());
+ final Range range = new Range(rootTableEnd.followingKey(PartialKey.ROW), true, Constants.METADATA_RESERVED_KEYSPACE_START_KEY, false);
+ final Scanner scanner = opts.getConnector().createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ scanner.setRange(range);
+ final Configuration conf = new Configuration();
+ final FileSystem fs = FileSystem.get(conf);
+
+ KeyExtent last = new KeyExtent();
+ String directory = null;
+ Set<String> knownFiles = new HashSet<String>();
+
+ int count = 0;
+ final MultiTableBatchWriter writer = opts.getConnector().createMultiTableBatchWriter(bwOpts.getBatchWriterConfig());
+
+ // collect the list of known files and the directory for each extent
+ for (Entry<Key,Value> entry : scanner) {
+ Key key = entry.getKey();
+ KeyExtent ke = new KeyExtent(key.getRow(), (Text) null);
+ // when the key extent changes
+ if (!ke.equals(last)) {
+ if (directory != null) {
+ // add any files in the directory unknown to the key extent
+ count += addUnknownFiles(fs, directory, knownFiles, last, writer, opts.update);
+ }
+ directory = null;
+ knownFiles.clear();
+ last = ke;
+ }
+ if (Constants.METADATA_DIRECTORY_COLUMN.hasColumns(key)) {
+ directory = entry.getValue().toString();
+ log.debug("Found directory " + directory + " for row " + key.getRow().toString());
+ } else if (key.compareColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY) == 0) {
+ String filename = key.getColumnQualifier().toString();
+ knownFiles.add(filename);
+ log.debug("METADATA file found: " + filename);
+ }
+ }
+ if (directory != null) {
+ // catch the last key extent
+ count += addUnknownFiles(fs, directory, knownFiles, last, writer, opts.update);
+ }
+ log.info("There were " + count + " files that are unknown to the metadata table");
+ writer.close();
+ }
+
+ private static int addUnknownFiles(FileSystem fs, String directory, Set<String> knownFiles, KeyExtent ke, MultiTableBatchWriter writer, boolean update) throws Exception {
+ int count = 0;
+ final String tableId = ke.getTableId().toString();
+ final Text row = ke.getMetadataEntry();
+ log.info(row.toString());
+ final Path path = new Path(ServerConstants.getTablesDir() + "/" + tableId + directory);
+ for (FileStatus file : fs.listStatus(path)) {
+ if (file.getPath().getName().endsWith("_tmp") || file.getPath().getName().endsWith("_tmp.rf"))
+ continue;
+ final String filename = directory + "/" + file.getPath().getName();
+ if (!knownFiles.contains(filename)) {
+ count++;
+ final Mutation m = new Mutation(row);
+ String size = Long.toString(file.getLen());
+ String entries = "1"; // lie
+ String value = size + "," + entries;
+ m.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(filename), new Value(value.getBytes(Constants.UTF8)));
+ if (update) {
+ writer.getBatchWriter(Constants.METADATA_TABLE_NAME).addMutation(m);
+ }
+ }
+ }
+ return count;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
index 95f6a32,0000000..3342993
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
@@@ -1,123 -1,0 +1,120 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+
+import com.beust.jcommander.Parameter;
+
+public class DumpZookeeper {
+
+ static IZooReaderWriter zk = null;
+
+ private static final Logger log = Logger.getLogger(DumpZookeeper.class);
+
+ private static class Encoded {
+ public String encoding;
+ public String value;
+
+ Encoded(String e, String v) {
+ encoding = e;
+ value = v;
+ }
+ }
+
+ static class Opts extends Help {
+ @Parameter(names="--root", description="the root of the znode tree to dump")
+ String root = "/";
+ }
+
- /**
- * @param args
- */
+ public static void main(String[] args) {
+ Opts opts = new Opts();
+ opts.parseArgs(DumpZookeeper.class.getName(), args);
+
+ Logger.getRootLogger().setLevel(Level.WARN);
+ PrintStream out = System.out;
+ try {
+ zk = ZooReaderWriter.getInstance();
+
+ write(out, 0, "<dump root='%s'>", opts.root);
+ for (String child : zk.getChildren(opts.root, null))
+ if (!child.equals("zookeeper"))
+ dump(out, opts.root, child, 1);
+ write(out, 0, "</dump>");
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ }
+
+ private static void dump(PrintStream out, String root, String child, int indent) throws KeeperException, InterruptedException, UnsupportedEncodingException {
+ String path = root + "/" + child;
+ if (root.endsWith("/"))
+ path = root + child;
+ Stat stat = zk.getStatus(path);
+ if (stat == null)
+ return;
+ String type = "node";
+ if (stat.getEphemeralOwner() != 0) {
+ type = "ephemeral";
+ }
+ if (stat.getNumChildren() == 0) {
+ if (stat.getDataLength() == 0) {
+ write(out, indent, "<%s name='%s'/>", type, child);
+ } else {
+ Encoded value = value(path);
+ write(out, indent, "<%s name='%s' encoding='%s' value='%s'/>", type, child, value.encoding, value.value);
+ }
+ } else {
+ if (stat.getDataLength() == 0) {
+ write(out, indent, "<%s name='%s'>", type, child);
+ } else {
+ Encoded value = value(path);
+ write(out, indent, "<%s name='%s' encoding='%s' value='%s'>", type, child, value.encoding, value.value);
+ }
+ for (String c : zk.getChildren(path, null)) {
+ dump(out, path, c, indent + 1);
+ }
+ write(out, indent, "</node>");
+ }
+ }
+
+ private static Encoded value(String path) throws KeeperException, InterruptedException, UnsupportedEncodingException {
+ byte[] data = zk.getData(path, null);
+ for (int i = 0; i < data.length; i++) {
+ // does this look like simple ascii?
+ if (data[i] < ' ' || data[i] > '~')
+ return new Encoded("base64", new String(Base64.encodeBase64(data), "utf8"));
+ }
+ return new Encoded("utf8", new String(data, "utf8"));
+ }
+
+ private static void write(PrintStream out, int indent, String fmt, Object... args) {
+ for (int i = 0; i < indent; i++)
+ out.print(" ");
+ out.println(String.format(fmt, args));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
index 60b50da,0000000..42ebbe2
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
@@@ -1,74 -1,0 +1,71 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.util.Iterator;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.server.master.LiveTServerSet;
+import org.apache.accumulo.server.master.LiveTServerSet.Listener;
+import org.apache.accumulo.server.master.state.MetaDataTableScanner;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletLocationState;
+import org.apache.accumulo.server.master.state.TabletState;
+import org.apache.accumulo.server.master.state.tables.TableManager;
+import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.commons.collections.iterators.IteratorChain;
+import org.apache.log4j.Logger;
+
+public class FindOfflineTablets {
+ private static final Logger log = Logger.getLogger(FindOfflineTablets.class);
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ ClientOpts opts = new ClientOpts();
+ opts.parseArgs(FindOfflineTablets.class.getName(), args);
+ final AtomicBoolean scanning = new AtomicBoolean(false);
+ Instance instance = opts.getInstance();
+ MetaDataTableScanner rootScanner = new MetaDataTableScanner(instance, SecurityConstants.getSystemCredentials(), Constants.METADATA_ROOT_TABLET_KEYSPACE);
+ MetaDataTableScanner metaScanner = new MetaDataTableScanner(instance, SecurityConstants.getSystemCredentials(), Constants.NON_ROOT_METADATA_KEYSPACE);
+ @SuppressWarnings("unchecked")
+ Iterator<TabletLocationState> scanner = (Iterator<TabletLocationState>)new IteratorChain(rootScanner, metaScanner);
+ LiveTServerSet tservers = new LiveTServerSet(instance, DefaultConfiguration.getDefaultConfiguration(), new Listener() {
+ @Override
+ public void update(LiveTServerSet current, Set<TServerInstance> deleted, Set<TServerInstance> added) {
+ if (!deleted.isEmpty() && scanning.get())
+ log.warn("Tablet servers deleted while scanning: " + deleted);
+ if (!added.isEmpty() && scanning.get())
+ log.warn("Tablet servers added while scanning: " + added);
+ }
+ });
+ tservers.startListeningForTabletServerChanges();
+ scanning.set(true);
+ while (scanner.hasNext()) {
+ TabletLocationState locationState = scanner.next();
+ TabletState state = locationState.getState(tservers.getCurrentServers());
+ if (state != null && state != TabletState.HOSTED && TableManager.getInstance().getTableState(locationState.extent.getTableId().toString()) != TableState.OFFLINE)
+ if (!locationState.extent.equals(Constants.ROOT_TABLET_EXTENT))
+ System.out.println(locationState + " is " + state + " #walogs:" + locationState.walogs.size());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
index e16bd06,0000000..cf1a065
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
@@@ -1,62 -1,0 +1,59 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.TokenProperty;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.security.handler.Authenticator;
+import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+
+/**
+ *
+ */
+public class LoginProperties {
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ AccumuloConfiguration config = ServerConfiguration.getSystemConfiguration(HdfsZooInstance.getInstance());
+ Authenticator authenticator = AccumuloClassLoader.getClassLoader().loadClass(config.get(Property.INSTANCE_SECURITY_AUTHENTICATOR))
+ .asSubclass(Authenticator.class).newInstance();
+
+ List<Set<TokenProperty>> tokenProps = new ArrayList<Set<TokenProperty>>();
+
+ for (Class<? extends AuthenticationToken> tokenType : authenticator.getSupportedTokenTypes()) {
+ tokenProps.add(tokenType.newInstance().getProperties());
+ }
+
+ System.out.println("Supported token types for " + authenticator.getClass().getName() + " are : ");
+ for (Class<? extends AuthenticationToken> tokenType : authenticator.getSupportedTokenTypes()) {
+ System.out.println("\t" + tokenType.getName() + ", which accepts the following properties : ");
+
+ for (TokenProperty tokenProperty : tokenType.newInstance().getProperties()) {
+ System.out.println("\t\t" + tokenProperty);
+ }
+
+ System.out.println();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
index d6e0a3c,0000000..477718d
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
@@@ -1,1262 -1,0 +1,1257 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.BatchWriterImpl;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
+import org.apache.accumulo.core.client.impl.Writer;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.accumulo.core.util.FastFormat;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.StringUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.accumulo.server.trace.TraceFileSystem;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * provides a reference to the metadata table for updates by tablet servers
+ */
+public class MetadataTable extends org.apache.accumulo.core.util.MetadataTable {
-
++
+ private static final Text EMPTY_TEXT = new Text();
+ private static Map<TCredentials,Writer> metadata_tables = new HashMap<TCredentials,Writer>();
+ private static final Logger log = Logger.getLogger(MetadataTable.class);
-
++
+ private static final int SAVE_ROOT_TABLET_RETRIES = 3;
-
++
+ private MetadataTable() {
-
++
+ }
+
+ public synchronized static Writer getMetadataTable(TCredentials credentials) {
+ Writer metadataTable = metadata_tables.get(credentials);
+ if (metadataTable == null) {
+ metadataTable = new Writer(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID);
+ metadata_tables.put(credentials, metadataTable);
+ }
+ return metadataTable;
+ }
-
++
+ public static void putLockID(ZooLock zooLock, Mutation m) {
+ Constants.METADATA_LOCK_COLUMN.put(m, new Value(zooLock.getLockID().serialize(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + "/").getBytes(Constants.UTF8)));
+ }
+
+ public static void update(TCredentials credentials, Mutation m) {
+ update(credentials, null, m);
+ }
+
+ public static void update(TCredentials credentials, ZooLock zooLock, Mutation m) {
+ Writer t;
+ t = getMetadataTable(credentials);
+ if (zooLock != null)
+ putLockID(zooLock, m);
+ while (true) {
+ try {
+ t.update(m);
+ return;
+ } catch (AccumuloException e) {
+ log.error(e, e);
+ } catch (AccumuloSecurityException e) {
+ log.error(e, e);
+ } catch (ConstraintViolationException e) {
+ log.error(e, e);
+ } catch (TableNotFoundException e) {
+ log.error(e, e);
+ }
+ UtilWaitThread.sleep(1000);
+ }
-
++
+ }
-
++
+ /**
+ * new data file update function adds one data file to a tablet's list
- *
- * path should be relative to the table directory
- *
- * @param time
- * @param filesInUseByScans
- * @param zooLock
- * @param flushId
- *
++ *
++ * @param path
++ * should be relative to the table directory
+ */
+ public static void updateTabletDataFile(KeyExtent extent, String path, String mergeFile, DataFileValue dfv, String time, TCredentials credentials,
+ Set<String> filesInUseByScans, String address, ZooLock zooLock, Set<String> unusedWalLogs, TServerInstance lastLocation, long flushId) {
+ if (extent.equals(Constants.ROOT_TABLET_EXTENT)) {
+ if (unusedWalLogs != null) {
+ IZooReaderWriter zk = ZooReaderWriter.getInstance();
+ // unusedWalLogs will contain the location/name of each log in a log set
+ // the log set is stored under one of the log names, but not both
+ // find the entry under one of the names and delete it.
+ String root = getZookeeperLogLocation();
+ boolean foundEntry = false;
+ for (String entry : unusedWalLogs) {
+ String[] parts = entry.split("/");
+ String zpath = root + "/" + parts[1];
+ while (true) {
+ try {
+ if (zk.exists(zpath)) {
+ zk.recursiveDelete(zpath, NodeMissingPolicy.SKIP);
+ foundEntry = true;
+ }
+ break;
+ } catch (KeeperException e) {
+ log.error(e, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ }
+ UtilWaitThread.sleep(1000);
+ }
+ }
+ if (unusedWalLogs.size() > 0 && !foundEntry)
+ log.warn("WALog entry for root tablet did not exist " + unusedWalLogs);
+ }
+ return;
+ }
-
++
+ Mutation m = new Mutation(extent.getMetadataEntry());
-
++
+ if (dfv.getNumEntries() > 0) {
+ m.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(path), new Value(dfv.encode()));
+ Constants.METADATA_TIME_COLUMN.put(m, new Value(time.getBytes(Constants.UTF8)));
+ // stuff in this location
+ TServerInstance self = getTServerInstance(address, zooLock);
+ self.putLastLocation(m);
+ // erase the old location
+ if (lastLocation != null && !lastLocation.equals(self))
+ lastLocation.clearLastLocation(m);
+ }
+ if (unusedWalLogs != null) {
+ for (String entry : unusedWalLogs) {
+ m.putDelete(Constants.METADATA_LOG_COLUMN_FAMILY, new Text(entry));
+ }
+ }
-
++
+ for (String scanFile : filesInUseByScans)
+ m.put(Constants.METADATA_SCANFILE_COLUMN_FAMILY, new Text(scanFile), new Value(new byte[0]));
+
+ if (mergeFile != null)
+ m.putDelete(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(mergeFile));
+
+ Constants.METADATA_FLUSH_COLUMN.put(m, new Value(Long.toString(flushId).getBytes(Constants.UTF8)));
+
+ update(credentials, zooLock, m);
-
++
+ }
-
++
+ private static TServerInstance getTServerInstance(String address, ZooLock zooLock) {
+ while (true) {
+ try {
+ return new TServerInstance(address, zooLock.getSessionId());
+ } catch (KeeperException e) {
+ log.error(e, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ }
+ UtilWaitThread.sleep(1000);
+ }
+ }
+
+ public static void updateTabletFlushID(KeyExtent extent, long flushID, TCredentials credentials, ZooLock zooLock) {
+ if (!extent.isRootTablet()) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
+ Constants.METADATA_FLUSH_COLUMN.put(m, new Value(Long.toString(flushID).getBytes(Constants.UTF8)));
+ update(credentials, zooLock, m);
+ }
+ }
+
+ public static void updateTabletCompactID(KeyExtent extent, long compactID, TCredentials credentials, ZooLock zooLock) {
+ if (!extent.isRootTablet()) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
+ Constants.METADATA_COMPACT_COLUMN.put(m, new Value(Long.toString(compactID).getBytes(Constants.UTF8)));
+ update(credentials, zooLock, m);
+ }
+ }
+
+ public static void updateTabletDataFile(long tid, KeyExtent extent, Map<String,DataFileValue> estSizes, String time, TCredentials credentials, ZooLock zooLock) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
+ byte[] tidBytes = Long.toString(tid).getBytes(Constants.UTF8);
+
+ for (Entry<String,DataFileValue> entry : estSizes.entrySet()) {
+ Text file = new Text(entry.getKey());
+ m.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, file, new Value(entry.getValue().encode()));
+ m.put(Constants.METADATA_BULKFILE_COLUMN_FAMILY, file, new Value(tidBytes));
+ }
+ Constants.METADATA_TIME_COLUMN.put(m, new Value(time.getBytes(Constants.UTF8)));
+ update(credentials, zooLock, m);
+ }
+
+ public static void addTablet(KeyExtent extent, String path, TCredentials credentials, char timeType, ZooLock lock) {
+ Mutation m = extent.getPrevRowUpdateMutation();
+
+ Constants.METADATA_DIRECTORY_COLUMN.put(m, new Value(path.getBytes(Constants.UTF8)));
+ Constants.METADATA_TIME_COLUMN.put(m, new Value((timeType + "0").getBytes(Constants.UTF8)));
+
+ update(credentials, lock, m);
+ }
+
+ public static void updateTabletPrevEndRow(KeyExtent extent, TCredentials credentials) {
+ Mutation m = extent.getPrevRowUpdateMutation(); //
+ update(credentials, m);
+ }
-
++
+ /**
+ * convenience method for reading entries from the metadata table
+ */
+ public static SortedMap<KeyExtent,Text> getMetadataDirectoryEntries(SortedMap<Key,Value> entries) {
+ Key key;
+ Value val;
+ Text datafile = null;
+ Value prevRow = null;
+ KeyExtent ke;
-
++
+ SortedMap<KeyExtent,Text> results = new TreeMap<KeyExtent,Text>();
-
++
+ Text lastRowFromKey = new Text();
-
++
+ // text obj below is meant to be reused in loop for efficiency
+ Text colf = new Text();
+ Text colq = new Text();
-
++
+ for (Entry<Key,Value> entry : entries.entrySet()) {
+ key = entry.getKey();
+ val = entry.getValue();
-
++
+ if (key.compareRow(lastRowFromKey) != 0) {
+ prevRow = null;
+ datafile = null;
+ key.getRow(lastRowFromKey);
+ }
-
++
+ colf = key.getColumnFamily(colf);
+ colq = key.getColumnQualifier(colq);
-
++
+ // interpret the row id as a key extent
+ if (Constants.METADATA_DIRECTORY_COLUMN.equals(colf, colq))
+ datafile = new Text(val.toString());
-
++
+ else if (Constants.METADATA_PREV_ROW_COLUMN.equals(colf, colq))
+ prevRow = new Value(val);
-
++
+ if (datafile != null && prevRow != null) {
+ ke = new KeyExtent(key.getRow(), prevRow);
+ results.put(ke, datafile);
-
++
+ datafile = null;
+ prevRow = null;
+ }
+ }
+ return results;
+ }
+
+ public static boolean recordRootTabletLocation(String address) {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ for (int i = 0; i < SAVE_ROOT_TABLET_RETRIES; i++) {
+ try {
+ log.info("trying to write root tablet location to ZooKeeper as " + address);
+ String zRootLocPath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZROOT_TABLET_LOCATION;
+ zoo.putPersistentData(zRootLocPath, address.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
+ return true;
+ } catch (Exception e) {
+ log.error("Master: unable to save root tablet location in zookeeper. exception: " + e, e);
+ }
+ }
+ log.error("Giving up after " + SAVE_ROOT_TABLET_RETRIES + " retries");
+ return false;
+ }
+
+ public static SortedMap<String,DataFileValue> getDataFileSizes(KeyExtent extent, TCredentials credentials) {
+ TreeMap<String,DataFileValue> sizes = new TreeMap<String,DataFileValue>();
-
++
+ Scanner mdScanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ mdScanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
+ Text row = extent.getMetadataEntry();
-
++
+ Key endKey = new Key(row, Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(""));
+ endKey = endKey.followingKey(PartialKey.ROW_COLFAM);
-
++
+ mdScanner.setRange(new Range(new Key(row), endKey));
+ for (Entry<Key,Value> entry : mdScanner) {
-
++
+ if (!entry.getKey().getRow().equals(row))
+ break;
+ DataFileValue dfv = new DataFileValue(entry.getValue().get());
+ sizes.put(entry.getKey().getColumnQualifier().toString(), dfv);
+ }
-
++
+ return sizes;
+ }
-
++
+ public static void addNewTablet(KeyExtent extent, String path, TServerInstance location, Map<String,DataFileValue> datafileSizes,
+ Map<String,Long> bulkLoadedFiles, TCredentials credentials, String time, long lastFlushID, long lastCompactID, ZooLock zooLock) {
+ Mutation m = extent.getPrevRowUpdateMutation();
+
+ Constants.METADATA_DIRECTORY_COLUMN.put(m, new Value(path.getBytes(Constants.UTF8)));
+ Constants.METADATA_TIME_COLUMN.put(m, new Value(time.getBytes(Constants.UTF8)));
+ if (lastFlushID > 0)
+ Constants.METADATA_FLUSH_COLUMN.put(m, new Value(Long.toString(lastFlushID).getBytes(Constants.UTF8)));
+ if (lastCompactID > 0)
+ Constants.METADATA_COMPACT_COLUMN.put(m, new Value(Long.toString(lastCompactID).getBytes(Constants.UTF8)));
+
+ if (location != null) {
+ m.put(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY, location.asColumnQualifier(), location.asMutationValue());
+ m.putDelete(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY, location.asColumnQualifier());
+ }
-
++
+ for (Entry<String,DataFileValue> entry : datafileSizes.entrySet()) {
+ m.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(entry.getKey()), new Value(entry.getValue().encode()));
+ }
-
++
+ for (Entry<String,Long> entry : bulkLoadedFiles.entrySet()) {
+ byte[] tidBytes = Long.toString(entry.getValue()).getBytes(Constants.UTF8);
+ m.put(Constants.METADATA_BULKFILE_COLUMN_FAMILY, new Text(entry.getKey()), new Value(tidBytes));
+ }
+
+ update(credentials, zooLock, m);
+ }
+
+ public static void rollBackSplit(Text metadataEntry, Text oldPrevEndRow, TCredentials credentials, ZooLock zooLock) {
+ KeyExtent ke = new KeyExtent(metadataEntry, oldPrevEndRow);
+ Mutation m = ke.getPrevRowUpdateMutation();
+ Constants.METADATA_SPLIT_RATIO_COLUMN.putDelete(m);
+ Constants.METADATA_OLD_PREV_ROW_COLUMN.putDelete(m);
+ update(credentials, zooLock, m);
+ }
+
+ public static void splitTablet(KeyExtent extent, Text oldPrevEndRow, double splitRatio, TCredentials credentials, ZooLock zooLock) {
+ Mutation m = extent.getPrevRowUpdateMutation(); //
+
+ Constants.METADATA_SPLIT_RATIO_COLUMN.put(m, new Value(Double.toString(splitRatio).getBytes(Constants.UTF8)));
+
+ Constants.METADATA_OLD_PREV_ROW_COLUMN.put(m, KeyExtent.encodePrevEndRow(oldPrevEndRow));
+ Constants.METADATA_CHOPPED_COLUMN.putDelete(m);
+ update(credentials, zooLock, m);
+ }
+
+ public static void finishSplit(Text metadataEntry, Map<String,DataFileValue> datafileSizes, List<String> highDatafilesToRemove, TCredentials credentials,
+ ZooLock zooLock) {
+ Mutation m = new Mutation(metadataEntry);
+ Constants.METADATA_SPLIT_RATIO_COLUMN.putDelete(m);
+ Constants.METADATA_OLD_PREV_ROW_COLUMN.putDelete(m);
+ Constants.METADATA_CHOPPED_COLUMN.putDelete(m);
+
+ for (Entry<String,DataFileValue> entry : datafileSizes.entrySet()) {
+ m.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(entry.getKey()), new Value(entry.getValue().encode()));
+ }
-
++
+ for (String pathToRemove : highDatafilesToRemove) {
+ m.putDelete(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(pathToRemove));
+ }
-
++
+ update(credentials, zooLock, m);
+ }
+
+ public static void finishSplit(KeyExtent extent, Map<String,DataFileValue> datafileSizes, List<String> highDatafilesToRemove, TCredentials credentials,
+ ZooLock zooLock) {
+ finishSplit(extent.getMetadataEntry(), datafileSizes, highDatafilesToRemove, credentials, zooLock);
+ }
-
++
+ public static void replaceDatafiles(KeyExtent extent, Set<String> datafilesToDelete, Set<String> scanFiles, String path, Long compactionId,
+ DataFileValue size, TCredentials credentials, String address, TServerInstance lastLocation, ZooLock zooLock) {
+ replaceDatafiles(extent, datafilesToDelete, scanFiles, path, compactionId, size, credentials, address, lastLocation, zooLock, true);
+ }
-
++
+ public static void replaceDatafiles(KeyExtent extent, Set<String> datafilesToDelete, Set<String> scanFiles, String path, Long compactionId,
+ DataFileValue size, TCredentials credentials, String address, TServerInstance lastLocation, ZooLock zooLock, boolean insertDeleteFlags) {
+
+ if (insertDeleteFlags) {
+ // add delete flags for those paths before the data file reference is removed
+ addDeleteEntries(extent, datafilesToDelete, credentials);
+ }
-
++
+ // replace data file references to old mapfiles with the new mapfiles
+ Mutation m = new Mutation(extent.getMetadataEntry());
-
++
+ for (String pathToRemove : datafilesToDelete)
+ m.putDelete(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(pathToRemove));
-
++
+ for (String scanFile : scanFiles)
+ m.put(Constants.METADATA_SCANFILE_COLUMN_FAMILY, new Text(scanFile), new Value(new byte[0]));
+
+ if (size.getNumEntries() > 0)
+ m.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(path), new Value(size.encode()));
-
++
+ if (compactionId != null)
+ Constants.METADATA_COMPACT_COLUMN.put(m, new Value(Long.toString(compactionId).getBytes(Constants.UTF8)));
+
+ TServerInstance self = getTServerInstance(address, zooLock);
+ self.putLastLocation(m);
+
+ // remove the old location
+ if (lastLocation != null && !lastLocation.equals(self))
+ lastLocation.clearLastLocation(m);
+
+ update(credentials, zooLock, m);
+ }
+
+ public static void addDeleteEntries(KeyExtent extent, Set<String> datafilesToDelete, TCredentials credentials) {
+
+ String tableId = extent.getTableId().toString();
+
+ // TODO could use batch writer,would need to handle failure and retry like update does - ACCUMULO-1294
+ for (String pathToRemove : datafilesToDelete)
+ update(credentials, createDeleteMutation(tableId, pathToRemove));
+ }
-
++
+ public static void addDeleteEntry(String tableId, String path) {
+ update(SecurityConstants.getSystemCredentials(), createDeleteMutation(tableId, path));
+ }
-
++
+ public static Mutation createDeleteMutation(String tableId, String pathToRemove) {
+ Mutation delFlag;
+ String prefix = Constants.METADATA_DELETE_FLAG_PREFIX;
+ if (tableId.equals(Constants.METADATA_TABLE_ID))
+ prefix = Constants.METADATA_DELETE_FLAG_FOR_METADATA_PREFIX;
+
+ if (pathToRemove.startsWith("../"))
+ delFlag = new Mutation(new Text(prefix + pathToRemove.substring(2)));
+ else
+ delFlag = new Mutation(new Text(prefix + "/" + tableId + pathToRemove));
+
+ delFlag.put(EMPTY_TEXT, EMPTY_TEXT, new Value(new byte[] {}));
+ return delFlag;
+ }
+
+ public static void removeScanFiles(KeyExtent extent, Set<String> scanFiles, TCredentials credentials, ZooLock zooLock) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
-
++
+ for (String pathToRemove : scanFiles)
+ m.putDelete(Constants.METADATA_SCANFILE_COLUMN_FAMILY, new Text(pathToRemove));
-
++
+ update(credentials, zooLock, m);
+ }
+
+ private static KeyExtent fixSplit(Text table, Text metadataEntry, Text metadataPrevEndRow, Value oper, double splitRatio, TServerInstance tserver,
+ TCredentials credentials, String time, long initFlushID, long initCompactID, ZooLock lock) throws AccumuloException {
+ if (metadataPrevEndRow == null)
+ // something is wrong, this should not happen... if a tablet is split, it will always have a
+ // prev end row....
+ throw new AccumuloException("Split tablet does not have prev end row, something is amiss, extent = " + metadataEntry);
+
+ // check to see if prev tablet exist in metadata tablet
+ Key prevRowKey = new Key(new Text(KeyExtent.getMetadataEntry(table, metadataPrevEndRow)));
+
+ ScannerImpl scanner2 = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ scanner2.setRange(new Range(prevRowKey, prevRowKey.followingKey(PartialKey.ROW)));
-
++
+ if (!scanner2.iterator().hasNext()) {
+ log.info("Rolling back incomplete split " + metadataEntry + " " + metadataPrevEndRow);
+ rollBackSplit(metadataEntry, KeyExtent.decodePrevEndRow(oper), credentials, lock);
+ return new KeyExtent(metadataEntry, KeyExtent.decodePrevEndRow(oper));
+ } else {
+ log.info("Finishing incomplete split " + metadataEntry + " " + metadataPrevEndRow);
+
+ List<String> highDatafilesToRemove = new ArrayList<String>();
+
+ Scanner scanner3 = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ Key rowKey = new Key(metadataEntry);
+
+ SortedMap<String,DataFileValue> origDatafileSizes = new TreeMap<String,DataFileValue>();
+ SortedMap<String,DataFileValue> highDatafileSizes = new TreeMap<String,DataFileValue>();
+ SortedMap<String,DataFileValue> lowDatafileSizes = new TreeMap<String,DataFileValue>();
+ scanner3.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
+ scanner3.setRange(new Range(rowKey, rowKey.followingKey(PartialKey.ROW)));
+
+ for (Entry<Key,Value> entry : scanner3) {
+ if (entry.getKey().compareColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY) == 0) {
+ origDatafileSizes.put(entry.getKey().getColumnQualifier().toString(), new DataFileValue(entry.getValue().get()));
+ }
+ }
+
+ splitDatafiles(table, metadataPrevEndRow, splitRatio, new HashMap<String,FileUtil.FileInfo>(), origDatafileSizes, lowDatafileSizes, highDatafileSizes,
+ highDatafilesToRemove);
+
+ MetadataTable.finishSplit(metadataEntry, highDatafileSizes, highDatafilesToRemove, credentials, lock);
+
+ return new KeyExtent(metadataEntry, KeyExtent.encodePrevEndRow(metadataPrevEndRow));
+ }
+
+
+ }
-
++
+ public static void splitDatafiles(Text table, Text midRow, double splitRatio, Map<String,FileUtil.FileInfo> firstAndLastRows,
+ SortedMap<String,DataFileValue> datafiles, SortedMap<String,DataFileValue> lowDatafileSizes, SortedMap<String,DataFileValue> highDatafileSizes,
+ List<String> highDatafilesToRemove) {
-
++
+ for (Entry<String,DataFileValue> entry : datafiles.entrySet()) {
-
++
+ Text firstRow = null;
+ Text lastRow = null;
-
++
+ boolean rowsKnown = false;
-
++
+ FileUtil.FileInfo mfi = firstAndLastRows.get(entry.getKey());
-
++
+ if (mfi != null) {
+ firstRow = mfi.getFirstRow();
+ lastRow = mfi.getLastRow();
+ rowsKnown = true;
+ }
-
++
+ if (rowsKnown && firstRow.compareTo(midRow) > 0) {
+ // only in high
+ long highSize = entry.getValue().getSize();
+ long highEntries = entry.getValue().getNumEntries();
+ highDatafileSizes.put(entry.getKey(), new DataFileValue(highSize, highEntries, entry.getValue().getTime()));
+ } else if (rowsKnown && lastRow.compareTo(midRow) <= 0) {
+ // only in low
+ long lowSize = entry.getValue().getSize();
+ long lowEntries = entry.getValue().getNumEntries();
+ lowDatafileSizes.put(entry.getKey(), new DataFileValue(lowSize, lowEntries, entry.getValue().getTime()));
-
++
+ highDatafilesToRemove.add(entry.getKey());
+ } else {
+ long lowSize = (long) Math.floor((entry.getValue().getSize() * splitRatio));
+ long lowEntries = (long) Math.floor((entry.getValue().getNumEntries() * splitRatio));
+ lowDatafileSizes.put(entry.getKey(), new DataFileValue(lowSize, lowEntries, entry.getValue().getTime()));
-
++
+ long highSize = (long) Math.ceil((entry.getValue().getSize() * (1.0 - splitRatio)));
+ long highEntries = (long) Math.ceil((entry.getValue().getNumEntries() * (1.0 - splitRatio)));
+ highDatafileSizes.put(entry.getKey(), new DataFileValue(highSize, highEntries, entry.getValue().getTime()));
+ }
+ }
+ }
+
+ public static KeyExtent fixSplit(Text metadataEntry, SortedMap<ColumnFQ,Value> columns, TServerInstance tserver, TCredentials credentials, ZooLock lock)
+ throws AccumuloException {
+ log.info("Incomplete split " + metadataEntry + " attempting to fix");
+
+ Value oper = columns.get(Constants.METADATA_OLD_PREV_ROW_COLUMN);
-
++
+ if (columns.get(Constants.METADATA_SPLIT_RATIO_COLUMN) == null) {
+ throw new IllegalArgumentException("Metadata entry does not have split ratio (" + metadataEntry + ")");
+ }
+
+ double splitRatio = Double.parseDouble(new String(columns.get(Constants.METADATA_SPLIT_RATIO_COLUMN).get(), Constants.UTF8));
+
+ Value prevEndRowIBW = columns.get(Constants.METADATA_PREV_ROW_COLUMN);
-
++
+ if (prevEndRowIBW == null) {
+ throw new IllegalArgumentException("Metadata entry does not have prev row (" + metadataEntry + ")");
+ }
-
++
+ Value time = columns.get(Constants.METADATA_TIME_COLUMN);
-
++
+ if (time == null) {
+ throw new IllegalArgumentException("Metadata entry does not have time (" + metadataEntry + ")");
+ }
-
++
+ Value flushID = columns.get(Constants.METADATA_FLUSH_COLUMN);
+ long initFlushID = -1;
+ if (flushID != null)
+ initFlushID = Long.parseLong(flushID.toString());
-
++
+ Value compactID = columns.get(Constants.METADATA_COMPACT_COLUMN);
+ long initCompactID = -1;
+ if (compactID != null)
+ initCompactID = Long.parseLong(compactID.toString());
-
++
+ Text metadataPrevEndRow = KeyExtent.decodePrevEndRow(prevEndRowIBW);
-
++
+ Text table = (new KeyExtent(metadataEntry, (Text) null)).getTableId();
-
++
+ return fixSplit(table, metadataEntry, metadataPrevEndRow, oper, splitRatio, tserver, credentials, time.toString(), initFlushID, initCompactID, lock);
+ }
+
+ public static void deleteTable(String tableId, boolean insertDeletes, TCredentials credentials, ZooLock lock) throws AccumuloException {
+ Scanner ms = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ Text tableIdText = new Text(tableId);
+ BatchWriter bw = new BatchWriterImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, new BatchWriterConfig().setMaxMemory(1000000)
+ .setMaxLatency(120000l, TimeUnit.MILLISECONDS).setMaxWriteThreads(2));
+
+ // scan metadata for our table and delete everything we find
+ Mutation m = null;
+ ms.setRange(new KeyExtent(tableIdText, null, null).toMetadataRange());
-
++
+ // insert deletes before deleting data from !METADATA... this makes the code fault tolerant
+ if (insertDeletes) {
-
++
+ ms.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
+ Constants.METADATA_DIRECTORY_COLUMN.fetch(ms);
+
+ for (Entry<Key,Value> cell : ms) {
+ Key key = cell.getKey();
-
++
+ if (key.getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
+ String relPath = key.getColumnQualifier().toString();
+ // only insert deletes for files owned by this table
+ if (!relPath.startsWith("../"))
+ bw.addMutation(createDeleteMutation(tableId, relPath));
+ }
-
++
+ if (Constants.METADATA_DIRECTORY_COLUMN.hasColumns(key)) {
+ bw.addMutation(createDeleteMutation(tableId, cell.getValue().toString()));
+ }
+ }
-
++
+ bw.flush();
-
++
+ ms.clearColumns();
+ }
-
++
+ for (Entry<Key,Value> cell : ms) {
+ Key key = cell.getKey();
-
++
+ if (m == null) {
+ m = new Mutation(key.getRow());
+ if (lock != null)
+ putLockID(lock, m);
+ }
-
++
+ if (key.getRow().compareTo(m.getRow(), 0, m.getRow().length) != 0) {
+ bw.addMutation(m);
+ m = new Mutation(key.getRow());
+ if (lock != null)
+ putLockID(lock, m);
+ }
+ m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
+ }
-
++
+ if (m != null)
+ bw.addMutation(m);
-
++
+ bw.close();
+ }
-
++
+ public static class LogEntry {
+ public KeyExtent extent;
+ public long timestamp;
+ public String server;
+ public String filename;
+ public int tabletId;
+ public Collection<String> logSet;
+
+ @Override
+ public String toString() {
+ return extent.toString() + " " + filename + " (" + tabletId + ")";
+ }
-
++
+ public String getName() {
+ return server + "/" + filename;
+ }
-
++
+ public byte[] toBytes() throws IOException {
+ DataOutputBuffer out = new DataOutputBuffer();
+ extent.write(out);
+ out.writeLong(timestamp);
+ out.writeUTF(server);
+ out.writeUTF(filename);
+ out.write(tabletId);
+ out.write(logSet.size());
+ for (String s : logSet) {
+ out.writeUTF(s);
+ }
+ return Arrays.copyOf(out.getData(), out.getLength());
+ }
-
++
+ public void fromBytes(byte bytes[]) throws IOException {
+ DataInputBuffer inp = new DataInputBuffer();
+ inp.reset(bytes, bytes.length);
+ extent = new KeyExtent();
+ extent.readFields(inp);
+ timestamp = inp.readLong();
+ server = inp.readUTF();
+ filename = inp.readUTF();
+ tabletId = inp.read();
+ int count = inp.read();
+ ArrayList<String> logSet = new ArrayList<String>(count);
+ for (int i = 0; i < count; i++)
+ logSet.add(inp.readUTF());
+ this.logSet = logSet;
+ }
-
++
+ }
-
++
+ private static String getZookeeperLogLocation() {
+ return ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZROOT_TABLET_WALOGS;
+ }
+
+ public static void addLogEntry(TCredentials credentials, LogEntry entry, ZooLock zooLock) {
+ if (entry.extent.isRootTablet()) {
+ String root = getZookeeperLogLocation();
+ while (true) {
+ try {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ if (zoo.isLockHeld(zooLock.getLockID()))
+ zoo.putPersistentData(root + "/" + entry.filename, entry.toBytes(), NodeExistsPolicy.OVERWRITE);
+ break;
+ } catch (KeeperException e) {
+ log.error(e, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ } catch (IOException e) {
+ log.error(e, e);
+ }
+ UtilWaitThread.sleep(1000);
+ }
+ } else {
+ String value = StringUtil.join(entry.logSet, ";") + "|" + entry.tabletId;
+ Mutation m = new Mutation(entry.extent.getMetadataEntry());
+ m.put(Constants.METADATA_LOG_COLUMN_FAMILY, new Text(entry.server + "/" + entry.filename), new Value(value.getBytes(Constants.UTF8)));
+ update(credentials, zooLock, m);
+ }
+ }
-
++
+ public static LogEntry entryFromKeyValue(Key key, Value value) {
+ MetadataTable.LogEntry e = new MetadataTable.LogEntry();
+ e.extent = new KeyExtent(key.getRow(), EMPTY_TEXT);
+ String[] parts = key.getColumnQualifier().toString().split("/");
+ e.server = parts[0];
+ e.filename = parts[1];
+ parts = value.toString().split("\\|");
+ e.tabletId = Integer.parseInt(parts[1]);
+ e.logSet = Arrays.asList(parts[0].split(";"));
+ e.timestamp = key.getTimestamp();
+ return e;
+ }
+
+ public static Pair<List<LogEntry>,SortedMap<String,DataFileValue>> getFileAndLogEntries(TCredentials credentials, KeyExtent extent) throws KeeperException,
+ InterruptedException, IOException {
+ ArrayList<LogEntry> result = new ArrayList<LogEntry>();
+ TreeMap<String,DataFileValue> sizes = new TreeMap<String,DataFileValue>();
+
+ if (extent.isRootTablet()) {
+ getRootLogEntries(result);
+ FileSystem fs = TraceFileSystem.wrap(FileUtil.getFileSystem(CachedConfiguration.getInstance(), ServerConfiguration.getSiteConfiguration()));
+ FileStatus[] files = fs.listStatus(new Path(ServerConstants.getRootTabletDir()));
-
++
+ for (FileStatus fileStatus : files) {
+ if (fileStatus.getPath().toString().endsWith("_tmp")) {
+ continue;
+ }
+ DataFileValue dfv = new DataFileValue(0, 0);
+ sizes.put(Constants.ZROOT_TABLET + "/" + fileStatus.getPath().getName(), dfv);
+ }
-
++
+ } else {
+ Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ scanner.fetchColumnFamily(Constants.METADATA_LOG_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
+ scanner.setRange(extent.toMetadataRange());
-
++
+ for (Entry<Key,Value> entry : scanner) {
+ if (!entry.getKey().getRow().equals(extent.getMetadataEntry())) {
+ throw new RuntimeException("Unexpected row " + entry.getKey().getRow() + " expected " + extent.getMetadataEntry());
+ }
-
++
+ if (entry.getKey().getColumnFamily().equals(Constants.METADATA_LOG_COLUMN_FAMILY)) {
+ result.add(entryFromKeyValue(entry.getKey(), entry.getValue()));
+ } else if (entry.getKey().getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
+ DataFileValue dfv = new DataFileValue(entry.getValue().get());
+ sizes.put(entry.getKey().getColumnQualifier().toString(), dfv);
+ } else {
+ throw new RuntimeException("Unexpected col fam " + entry.getKey().getColumnFamily());
+ }
+ }
+ }
-
++
+ return new Pair<List<LogEntry>,SortedMap<String,DataFileValue>>(result, sizes);
+ }
+
+ public static List<LogEntry> getLogEntries(TCredentials credentials, KeyExtent extent) throws IOException, KeeperException, InterruptedException {
+ log.info("Scanning logging entries for " + extent);
+ ArrayList<LogEntry> result = new ArrayList<LogEntry>();
+ if (extent.equals(Constants.ROOT_TABLET_EXTENT)) {
+ log.info("Getting logs for root tablet from zookeeper");
+ getRootLogEntries(result);
+ } else {
+ log.info("Scanning metadata for logs used for tablet " + extent);
+ Scanner scanner = getTabletLogScanner(credentials, extent);
+ Text pattern = extent.getMetadataEntry();
+ for (Entry<Key,Value> entry : scanner) {
+ Text row = entry.getKey().getRow();
+ if (entry.getKey().getColumnFamily().equals(Constants.METADATA_LOG_COLUMN_FAMILY)) {
+ if (row.equals(pattern)) {
+ result.add(entryFromKeyValue(entry.getKey(), entry.getValue()));
+ }
+ }
+ }
+ }
-
++
+ Collections.sort(result, new Comparator<LogEntry>() {
+ @Override
+ public int compare(LogEntry o1, LogEntry o2) {
+ long diff = o1.timestamp - o2.timestamp;
+ if (diff < 0)
+ return -1;
+ if (diff > 0)
+ return 1;
+ return 0;
+ }
+ });
+ log.info("Returning logs " + result + " for extent " + extent);
+ return result;
+ }
-
++
+ private static void getRootLogEntries(ArrayList<LogEntry> result) throws KeeperException, InterruptedException, IOException {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ String root = getZookeeperLogLocation();
+ // there's a little race between getting the children and fetching
+ // the data. The log can be removed in between.
+ while (true) {
+ result.clear();
+ for (String child : zoo.getChildren(root)) {
+ LogEntry e = new LogEntry();
+ try {
+ e.fromBytes(zoo.getData(root + "/" + child, null));
+ result.add(e);
+ } catch (KeeperException.NoNodeException ex) {
+ continue;
+ }
+ }
+ break;
+ }
+ }
+
+ private static Scanner getTabletLogScanner(TCredentials credentials, KeyExtent extent) {
+ Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ scanner.fetchColumnFamily(Constants.METADATA_LOG_COLUMN_FAMILY);
+ Text start = extent.getMetadataEntry();
+ Key endKey = new Key(start, Constants.METADATA_LOG_COLUMN_FAMILY);
+ endKey = endKey.followingKey(PartialKey.ROW_COLFAM);
+ scanner.setRange(new Range(new Key(start), endKey));
+ return scanner;
+ }
-
++
+ static class LogEntryIterator implements Iterator<LogEntry> {
-
++
+ Iterator<LogEntry> rootTabletEntries = null;
+ Iterator<Entry<Key,Value>> metadataEntries = null;
+
+ LogEntryIterator(TCredentials creds) throws IOException, KeeperException, InterruptedException {
+ rootTabletEntries = getLogEntries(creds, Constants.ROOT_TABLET_EXTENT).iterator();
+ try {
+ Scanner scanner = HdfsZooInstance.getInstance().getConnector(creds.getPrincipal(), CredentialHelper.extractToken(creds))
+ .createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ scanner.fetchColumnFamily(Constants.METADATA_LOG_COLUMN_FAMILY);
+ metadataEntries = scanner.iterator();
+ } catch (Exception ex) {
+ throw new IOException(ex);
+ }
+ }
-
++
+ @Override
+ public boolean hasNext() {
+ return rootTabletEntries.hasNext() || metadataEntries.hasNext();
+ }
-
++
+ @Override
+ public LogEntry next() {
+ if (rootTabletEntries.hasNext()) {
+ return rootTabletEntries.next();
+ }
+ Entry<Key,Value> entry = metadataEntries.next();
+ return entryFromKeyValue(entry.getKey(), entry.getValue());
+ }
-
++
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ public static Iterator<LogEntry> getLogEntries(TCredentials creds) throws IOException, KeeperException, InterruptedException {
+ return new LogEntryIterator(creds);
+ }
-
++
+ public static void removeUnusedWALEntries(KeyExtent extent, List<LogEntry> logEntries, ZooLock zooLock) {
+ if (extent.isRootTablet()) {
+ for (LogEntry entry : logEntries) {
+ String root = getZookeeperLogLocation();
+ while (true) {
+ try {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ if (zoo.isLockHeld(zooLock.getLockID()))
+ zoo.recursiveDelete(root + "/" + entry.filename, NodeMissingPolicy.SKIP);
+ break;
+ } catch (Exception e) {
+ log.error(e, e);
+ }
+ UtilWaitThread.sleep(1000);
+ }
+ }
+ } else {
+ Mutation m = new Mutation(extent.getMetadataEntry());
+ for (LogEntry entry : logEntries) {
+ m.putDelete(Constants.METADATA_LOG_COLUMN_FAMILY, new Text(entry.server + "/" + entry.filename));
+ }
+ update(SecurityConstants.getSystemCredentials(), zooLock, m);
+ }
+ }
-
++
+ private static void getFiles(Set<String> files, Map<Key,Value> tablet, String srcTableId) {
+ for (Entry<Key,Value> entry : tablet.entrySet()) {
+ if (entry.getKey().getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
+ String cf = entry.getKey().getColumnQualifier().toString();
+ if (srcTableId != null && !cf.startsWith("../"))
+ cf = "../" + srcTableId + entry.getKey().getColumnQualifier();
+ files.add(cf);
+ }
+ }
+ }
-
++
+ private static Mutation createCloneMutation(String srcTableId, String tableId, Map<Key,Value> tablet) {
-
++
+ KeyExtent ke = new KeyExtent(tablet.keySet().iterator().next().getRow(), (Text) null);
+ Mutation m = new Mutation(KeyExtent.getMetadataEntry(new Text(tableId), ke.getEndRow()));
-
++
+ for (Entry<Key,Value> entry : tablet.entrySet()) {
+ if (entry.getKey().getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
+ String cf = entry.getKey().getColumnQualifier().toString();
+ if (!cf.startsWith("../"))
+ cf = "../" + srcTableId + entry.getKey().getColumnQualifier();
+ m.put(entry.getKey().getColumnFamily(), new Text(cf), entry.getValue());
+ } else if (entry.getKey().getColumnFamily().equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY)) {
+ m.put(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY, entry.getKey().getColumnQualifier(), entry.getValue());
+ } else if (entry.getKey().getColumnFamily().equals(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY)) {
+ // skip
+ } else {
+ m.put(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier(), entry.getValue());
+ }
+ }
+ return m;
+ }
-
++
+ private static Scanner createCloneScanner(String tableId, Connector conn) throws TableNotFoundException {
+ Scanner mscanner = new IsolatedScanner(conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS));
+ mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+ mscanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
+ mscanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
+ mscanner.fetchColumnFamily(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY);
+ mscanner.fetchColumnFamily(Constants.METADATA_CLONED_COLUMN_FAMILY);
+ Constants.METADATA_PREV_ROW_COLUMN.fetch(mscanner);
+ Constants.METADATA_TIME_COLUMN.fetch(mscanner);
+ return mscanner;
+ }
-
++
+ static void initializeClone(String srcTableId, String tableId, Connector conn, BatchWriter bw) throws TableNotFoundException, MutationsRejectedException {
+ TabletIterator ti = new TabletIterator(createCloneScanner(srcTableId, conn), new KeyExtent(new Text(srcTableId), null, null).toMetadataRange(), true, true);
-
++
+ if (!ti.hasNext())
+ throw new RuntimeException(" table deleted during clone? srcTableId = " + srcTableId);
-
++
+ while (ti.hasNext())
+ bw.addMutation(createCloneMutation(srcTableId, tableId, ti.next()));
-
++
+ bw.flush();
+ }
-
++
+ static int compareEndRows(Text endRow1, Text endRow2) {
+ return new KeyExtent(new Text("0"), endRow1, null).compareTo(new KeyExtent(new Text("0"), endRow2, null));
+ }
-
++
+ static int checkClone(String srcTableId, String tableId, Connector conn, BatchWriter bw) throws TableNotFoundException, MutationsRejectedException {
+ TabletIterator srcIter = new TabletIterator(createCloneScanner(srcTableId, conn), new KeyExtent(new Text(srcTableId), null, null).toMetadataRange(), true,
+ true);
+ TabletIterator cloneIter = new TabletIterator(createCloneScanner(tableId, conn), new KeyExtent(new Text(tableId), null, null).toMetadataRange(), true, true);
-
++
+ if (!cloneIter.hasNext() || !srcIter.hasNext())
+ throw new RuntimeException(" table deleted during clone? srcTableId = " + srcTableId + " tableId=" + tableId);
-
++
+ int rewrites = 0;
-
++
+ while (cloneIter.hasNext()) {
+ Map<Key,Value> cloneTablet = cloneIter.next();
+ Text cloneEndRow = new KeyExtent(cloneTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
+ HashSet<String> cloneFiles = new HashSet<String>();
-
++
+ boolean cloneSuccessful = false;
+ for (Entry<Key,Value> entry : cloneTablet.entrySet()) {
+ if (entry.getKey().getColumnFamily().equals(Constants.METADATA_CLONED_COLUMN_FAMILY)) {
+ cloneSuccessful = true;
+ break;
+ }
+ }
-
++
+ if (!cloneSuccessful)
+ getFiles(cloneFiles, cloneTablet, null);
-
++
+ List<Map<Key,Value>> srcTablets = new ArrayList<Map<Key,Value>>();
+ Map<Key,Value> srcTablet = srcIter.next();
+ srcTablets.add(srcTablet);
-
++
+ Text srcEndRow = new KeyExtent(srcTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
-
++
+ int cmp = compareEndRows(cloneEndRow, srcEndRow);
+ if (cmp < 0)
+ throw new TabletIterator.TabletDeletedException("Tablets deleted from src during clone : " + cloneEndRow + " " + srcEndRow);
-
++
+ HashSet<String> srcFiles = new HashSet<String>();
+ if (!cloneSuccessful)
+ getFiles(srcFiles, srcTablet, srcTableId);
-
++
+ while (cmp > 0) {
+ srcTablet = srcIter.next();
+ srcTablets.add(srcTablet);
+ srcEndRow = new KeyExtent(srcTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
+ cmp = compareEndRows(cloneEndRow, srcEndRow);
+ if (cmp < 0)
+ throw new TabletIterator.TabletDeletedException("Tablets deleted from src during clone : " + cloneEndRow + " " + srcEndRow);
-
++
+ if (!cloneSuccessful)
+ getFiles(srcFiles, srcTablet, srcTableId);
+ }
-
++
+ if (cloneSuccessful)
+ continue;
-
++
+ if (!srcFiles.containsAll(cloneFiles)) {
+ // delete existing cloned tablet entry
+ Mutation m = new Mutation(cloneTablet.keySet().iterator().next().getRow());
-
++
+ for (Entry<Key,Value> entry : cloneTablet.entrySet()) {
+ Key k = entry.getKey();
+ m.putDelete(k.getColumnFamily(), k.getColumnQualifier(), k.getTimestamp());
+ }
-
++
+ bw.addMutation(m);
-
++
+ for (Map<Key,Value> st : srcTablets)
+ bw.addMutation(createCloneMutation(srcTableId, tableId, st));
-
++
+ rewrites++;
+ } else {
+ // write out marker that this tablet was successfully cloned
+ Mutation m = new Mutation(cloneTablet.keySet().iterator().next().getRow());
+ m.put(Constants.METADATA_CLONED_COLUMN_FAMILY, new Text(""), new Value("OK".getBytes(Constants.UTF8)));
+ bw.addMutation(m);
+ }
+ }
-
++
+ bw.flush();
+ return rewrites;
+ }
-
++
+ public static void cloneTable(Instance instance, String srcTableId, String tableId) throws Exception {
+
+ Connector conn = instance.getConnector(SecurityConstants.SYSTEM_PRINCIPAL, SecurityConstants.getSystemToken());
+ BatchWriter bw = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig());
+
+ while (true) {
-
++
+ try {
+ initializeClone(srcTableId, tableId, conn, bw);
-
++
+ // the following loop looks changes in the file that occurred during the copy.. if files were dereferenced then they could have been GCed
-
++
+ while (true) {
+ int rewrites = checkClone(srcTableId, tableId, conn, bw);
-
++
+ if (rewrites == 0)
+ break;
+ }
-
++
+ bw.flush();
+ break;
-
++
+ } catch (TabletIterator.TabletDeletedException tde) {
+ // tablets were merged in the src table
+ bw.flush();
-
++
+ // delete what we have cloned and try again
+ deleteTable(tableId, false, SecurityConstants.getSystemCredentials(), null);
-
++
+ log.debug("Tablets merged in table " + srcTableId + " while attempting to clone, trying again");
-
++
+ UtilWaitThread.sleep(100);
+ }
+ }
-
++
+ // delete the clone markers and create directory entries
+ Scanner mscanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+ mscanner.fetchColumnFamily(Constants.METADATA_CLONED_COLUMN_FAMILY);
-
++
+ int dirCount = 0;
-
++
+ for (Entry<Key,Value> entry : mscanner) {
+ Key k = entry.getKey();
+ Mutation m = new Mutation(k.getRow());
+ m.putDelete(k.getColumnFamily(), k.getColumnQualifier());
+ Constants.METADATA_DIRECTORY_COLUMN.put(m, new Value(FastFormat.toZeroPaddedString(dirCount++, 8, 16, "/c-".getBytes(Constants.UTF8))));
+ bw.addMutation(m);
+ }
-
++
+ bw.close();
-
++
+ }
-
++
+ public static void chopped(KeyExtent extent, ZooLock zooLock) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
+ Constants.METADATA_CHOPPED_COLUMN.put(m, new Value("chopped".getBytes(Constants.UTF8)));
+ update(SecurityConstants.getSystemCredentials(), zooLock, m);
+ }
-
++
+ public static void removeBulkLoadEntries(Connector conn, String tableId, long tid) throws Exception {
+ Scanner mscanner = new IsolatedScanner(conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS));
+ mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+ mscanner.fetchColumnFamily(Constants.METADATA_BULKFILE_COLUMN_FAMILY);
+ BatchWriter bw = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig());
+ for (Entry<Key,Value> entry : mscanner) {
+ log.debug("Looking at entry " + entry + " with tid " + tid);
+ if (Long.parseLong(entry.getValue().toString()) == tid) {
+ log.debug("deleting entry " + entry);
+ Mutation m = new Mutation(entry.getKey().getRow());
+ m.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier());
+ bw.addMutation(m);
+ }
+ }
+ bw.close();
+ }
-
++
+ public static List<String> getBulkFilesLoaded(Connector conn, KeyExtent extent, long tid) {
+ List<String> result = new ArrayList<String>();
+ try {
+ Scanner mscanner = new IsolatedScanner(conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS));
+ mscanner.setRange(extent.toMetadataRange());
+ mscanner.fetchColumnFamily(Constants.METADATA_BULKFILE_COLUMN_FAMILY);
+ for (Entry<Key,Value> entry : mscanner) {
+ if (Long.parseLong(entry.getValue().toString()) == tid) {
+ result.add(entry.getKey().getColumnQualifier().toString());
+ }
+ }
+ return result;
+ } catch (TableNotFoundException ex) {
+ // unlikely
+ throw new RuntimeException("Onos! teh metadata table has vanished!!");
+ }
+ }
+
+ public static Map<String,Long> getBulkFilesLoaded(TCredentials credentials, KeyExtent extent) {
+ return getBulkFilesLoaded(credentials, extent.getMetadataEntry());
+ }
+
+ public static Map<String,Long> getBulkFilesLoaded(TCredentials credentials, Text metadataRow) {
+
+ Map<String,Long> ret = new HashMap<String,Long>();
-
++
+ Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ scanner.setRange(new Range(metadataRow));
+ scanner.fetchColumnFamily(Constants.METADATA_BULKFILE_COLUMN_FAMILY);
+ for (Entry<Key,Value> entry : scanner) {
+ String file = entry.getKey().getColumnQualifier().toString();
+ Long tid = Long.parseLong(entry.getValue().toString());
-
++
+ ret.put(file, tid);
+ }
+ return ret;
+ }
+
+ public static void addBulkLoadInProgressFlag(String path) {
-
++
+ Mutation m = new Mutation(Constants.METADATA_BLIP_FLAG_PREFIX + path);
+ m.put(EMPTY_TEXT, EMPTY_TEXT, new Value(new byte[] {}));
-
++
+ update(SecurityConstants.getSystemCredentials(), m);
+ }
-
++
+ public static void removeBulkLoadInProgressFlag(String path) {
-
++
+ Mutation m = new Mutation(Constants.METADATA_BLIP_FLAG_PREFIX + path);
+ m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
-
++
+ update(SecurityConstants.getSystemCredentials(), m);
+ }
+
+ /**
+ * During an upgrade from Accumulo 1.4 -> 1.5, we need to move deletion requests for files under the !METADATA table to the root tablet.
+ */
+ public static void moveMetaDeleteMarkers(Instance instance, TCredentials creds) {
+ // move delete markers from the normal delete keyspace to the root tablet delete keyspace if the files are for the !METADATA table
+ Scanner scanner = new ScannerImpl(instance, creds, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ scanner.setRange(new Range(Constants.METADATA_DELETES_KEYSPACE));
+ for (Entry<Key,Value> entry : scanner) {
+ String row = entry.getKey().getRow().toString();
+ if (row.startsWith(Constants.METADATA_DELETE_FLAG_PREFIX + "/" + Constants.METADATA_TABLE_ID)) {
+ String filename = row.substring(Constants.METADATA_DELETE_FLAG_PREFIX.length());
+ // add the new entry first
+ log.info("Moving " + filename + " marker to the root tablet");
+ Mutation m = new Mutation(Constants.METADATA_DELETE_FLAG_FOR_METADATA_PREFIX + filename);
+ m.put(new byte[]{}, new byte[]{}, new byte[]{});
+ update(creds, m);
+ // remove the old entry
+ m = new Mutation(entry.getKey().getRow());
+ m.putDelete(new byte[]{}, new byte[]{});
+ update(creds, m);
+ } else {
+ break;
+ }
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
index 8985b27,0000000..8e41db7
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
@@@ -1,127 -1,0 +1,123 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.Stack;
+
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.DefaultHandler;
+
+import com.beust.jcommander.Parameter;
+
+public class RestoreZookeeper {
+
+ private static class Restore extends DefaultHandler {
+ IZooReaderWriter zk = null;
+ Stack<String> cwd = new Stack<String>();
+ boolean overwrite = false;
+
+ Restore(IZooReaderWriter zk, boolean overwrite) {
+ this.zk = zk;
+ this.overwrite = overwrite;
+ }
+
+ @Override
+ public void startElement(String uri, String localName, String name, Attributes attributes) throws SAXException {
+ if ("node".equals(name)) {
+ String child = attributes.getValue("name");
+ if (child == null)
+ throw new RuntimeException("name attribute not set");
+ String encoding = attributes.getValue("encoding");
+ String value = attributes.getValue("value");
+ if (value == null)
+ value = "";
+ String path = cwd.lastElement() + "/" + child;
+ create(path, value, encoding);
+ cwd.push(path);
+ } else if ("dump".equals(name)) {
+ String root = attributes.getValue("root");
+ if (root.equals("/"))
+ cwd.push("");
+ else
+ cwd.push(root);
+ create(root, "", "utf-8");
+ }
+ }
+
+ @Override
+ public void endElement(String uri, String localName, String name) throws SAXException {
+ cwd.pop();
+ }
+
+ private void create(String path, String value, String encoding) {
+ byte[] data = value.getBytes(Constants.UTF8);
+ if ("base64".equals(encoding))
+ data = Base64.decodeBase64(data);
+ try {
+ try {
+ zk.putPersistentData(path, data, overwrite ? NodeExistsPolicy.OVERWRITE : NodeExistsPolicy.FAIL);
+ } catch (KeeperException e) {
+ if (e.code().equals(KeeperException.Code.NODEEXISTS))
+ throw new RuntimeException(path + " exists. Remove it first.");
+ throw e;
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static class Opts extends Help {
+ @Parameter(names={"-z", "--keepers"})
+ String keepers = "localhost:2181";
+ @Parameter(names="--overwrite")
+ boolean overwrite = false;
+ @Parameter(names="--file")
+ String file;
+ }
+
- /**
- * @param args
- * @throws Exception
- */
+ public static void main(String[] args) throws Exception {
+ Logger.getRootLogger().setLevel(Level.WARN);
+ Opts opts = new Opts();
+ opts.parseArgs(RestoreZookeeper.class.getName(), args);
+
+ InputStream in = System.in;
+ if (opts.file != null) {
+ in = new FileInputStream(opts.file);
+ }
+
+ SAXParserFactory factory = SAXParserFactory.newInstance();
+ SAXParser parser = factory.newSAXParser();
+ parser.parse(in, new Restore(ZooReaderWriter.getInstance(), opts.overwrite));
+ in.close();
+ }
+}
[02/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
index efadfae,0000000..09fbbd2
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
@@@ -1,279 -1,0 +1,278 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
+import java.net.Socket;
+import java.net.URLEncoder;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.net.SocketFactory;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.commons.io.filefilter.WildcardFileFilter;
+import org.apache.commons.lang.math.LongRange;
+import org.apache.log4j.Category;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.Filter;
+import org.apache.log4j.spi.LocationInfo;
+import org.apache.log4j.spi.LoggingEvent;
+import org.apache.log4j.spi.ThrowableInformation;
+import org.apache.log4j.varia.LevelRangeFilter;
+import org.apache.log4j.xml.XMLLayout;
+
+import com.beust.jcommander.IStringConverter;
+import com.beust.jcommander.Parameter;
+
+public class SendLogToChainsaw extends XMLLayout {
+
+ private static Pattern logPattern = Pattern.compile(
+ "^(\\d\\d)\\s(\\d\\d):(\\d\\d):(\\d\\d),(\\d\\d\\d)\\s\\[(.*)\\]\\s(TRACE|DEBUG|INFO|WARN|FATAL|ERROR)\\s*?:(.*)$", Pattern.UNIX_LINES);
+
+ private File[] logFiles = null;
+
+ private SocketFactory factory = SocketFactory.getDefault();
+
+ private WildcardFileFilter fileFilter = null;
+
+ private Socket socket = null;
+
+ private Pattern lineFilter = null;
+
+ private LongRange dateFilter = null;
+
+ private LevelRangeFilter levelFilter = null;
+
+ public SendLogToChainsaw(String directory, String fileNameFilter, String host, int port, Date start, Date end, String regex, String level) throws Exception {
+
+ // Set up the file name filter
+ if (null != fileNameFilter) {
+ fileFilter = new WildcardFileFilter(fileNameFilter);
+ } else {
+ fileFilter = new WildcardFileFilter("*");
+ }
+
+ // Get the list of files that match
+ File dir = new File(directory);
+ if (dir.isDirectory()) {
+ logFiles = dir.listFiles((FilenameFilter) fileFilter);
+ } else {
+ throw new IllegalArgumentException(directory + " is not a directory or is not readable.");
+ }
+
+ if (logFiles.length == 0) {
+ throw new IllegalArgumentException("No files match the supplied filter.");
+ }
+
+ socket = factory.createSocket(host, port);
+
+ lineFilter = Pattern.compile(regex);
+
+ // Create Date Filter
+ if (null != start) {
+ if (end == null)
+ end = new Date(System.currentTimeMillis());
+ dateFilter = new LongRange(start.getTime(), end.getTime());
+ }
+
+ if (null != level) {
+ Level base = Level.toLevel(level.toUpperCase());
+ levelFilter = new LevelRangeFilter();
+ levelFilter.setAcceptOnMatch(true);
+ levelFilter.setLevelMin(base);
+ levelFilter.setLevelMax(Level.FATAL);
+ }
+ }
+
+ public void processLogFiles() throws IOException {
+ String line = null;
+ String out = null;
+ InputStreamReader isReader = null;
+ BufferedReader reader = null;
+ try {
+ for (File log : logFiles) {
+ // Parse the server type and name from the log file name
+ String threadName = log.getName().substring(0, log.getName().indexOf("."));
+ try {
+ isReader = new InputStreamReader(new FileInputStream(log), Constants.UTF8);
+ } catch (FileNotFoundException e) {
+ System.out.println("Unable to find file: " + log.getAbsolutePath());
+ throw e;
+ }
+ reader = new BufferedReader(isReader);
+
+ try {
+ line = reader.readLine();
+ while (null != line) {
+ out = convertLine(line, threadName);
+ if (null != out) {
+ if (socket != null && socket.isConnected())
+ socket.getOutputStream().write(out.getBytes(Constants.UTF8));
+ else
+ System.err.println("Unable to send data to transport");
+ }
+ line = reader.readLine();
+ }
+ } catch (IOException e) {
+ System.out.println("Error processing line: " + line + ". Output was " + out);
+ throw e;
+ } finally {
+ if (reader != null) {
+ reader.close();
+ }
+ if (isReader != null) {
+ isReader.close();
+ }
+ }
+ }
+ } finally {
+ if (socket != null && socket.isConnected()) {
+ socket.close();
+ }
+ }
+ }
+
+ private String convertLine(String line, String threadName) throws UnsupportedEncodingException {
+ String result = null;
+ Matcher m = logPattern.matcher(line);
+ if (m.matches()) {
+
+ Calendar cal = Calendar.getInstance();
+ cal.setTime(new Date(System.currentTimeMillis()));
+ Integer date = Integer.parseInt(m.group(1));
+ Integer hour = Integer.parseInt(m.group(2));
+ Integer min = Integer.parseInt(m.group(3));
+ Integer sec = Integer.parseInt(m.group(4));
+ Integer ms = Integer.parseInt(m.group(5));
+ String clazz = m.group(6);
+ String level = m.group(7);
+ String message = m.group(8);
+ // Apply the regex filter if supplied
+ if (null != lineFilter) {
+ Matcher match = lineFilter.matcher(message);
+ if (!match.matches())
+ return null;
+ }
+ // URL encode the message
+ message = URLEncoder.encode(message, "UTF-8");
+ // Assume that we are processing logs from today.
+ // If the date in the line is greater than today, then it must be
+ // from the previous month.
+ cal.set(Calendar.DATE, date);
+ cal.set(Calendar.HOUR_OF_DAY, hour);
+ cal.set(Calendar.MINUTE, min);
+ cal.set(Calendar.SECOND, sec);
+ cal.set(Calendar.MILLISECOND, ms);
+ if (date > cal.get(Calendar.DAY_OF_MONTH)) {
+ cal.add(Calendar.MONTH, -1);
+ }
+ long ts = cal.getTimeInMillis();
+ // If this event is not between the start and end dates, then skip it.
+ if (null != dateFilter && !dateFilter.containsLong(ts))
+ return null;
+ Category c = Logger.getLogger(clazz);
+ Level l = Level.toLevel(level);
+ LoggingEvent event = new LoggingEvent(clazz, c, ts, l, message, threadName, (ThrowableInformation) null, (String) null, (LocationInfo) null,
+ (Map<?,?>) null);
+ // Check the log level filter
+ if (null != levelFilter && (levelFilter.decide(event) == Filter.DENY)) {
+ return null;
+ }
+ result = format(event);
+ }
+ return result;
+ }
+
+ private static class DateConverter implements IStringConverter<Date> {
+ @Override
+ public Date convert(String value) {
+ SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
+ try {
+ return formatter.parse(value);
+ } catch (ParseException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+ private static class Opts extends Help {
+
+ @Parameter(names={"-d", "--logDirectory"}, description="ACCUMULO log directory path", required=true)
+ String dir;
+
+ @Parameter(names={"-f", "--fileFilter"}, description="filter to apply to names of logs")
+ String filter;
+
+ @Parameter(names={"-h", "--host"}, description="host where chainsaw is running", required=true)
+ String hostname;
+
+ @Parameter(names={"-p", "--port"}, description="port where XMLSocketReceiver is listening", required=true)
+ int portnum;
+
+ @Parameter(names={"-s", "--start"}, description="start date filter (yyyyMMddHHmmss)", required=true, converter=DateConverter.class)
+ Date startDate;
+
+ @Parameter(names={"-e", "--end"}, description="end date filter (yyyyMMddHHmmss)", required=true, converter=DateConverter.class)
+ Date endDate;
+
+ @Parameter(names={"-l", "--level"}, description="filter log level")
+ String level;
+
+ @Parameter(names={"-m", "--messageFilter"}, description="regex filter for log messages")
+ String regex;
+ }
+
+
+
+
+ /**
+ *
+ * @param args
+ * 0: path to log directory parameter
+ * 1: filter to apply for logs to include (uses wildcards (i.e. logger* and IS case sensitive) parameter
+ * 2: chainsaw host parameter
+ * 3: chainsaw port parameter
+ * 4: start date filter parameter
+ * 5: end date filter parameter
+ * 6: optional regex filter to match on each log4j message parameter
+ * 7: optional level filter
- * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(SendLogToChainsaw.class.getName(), args);
+
+ SendLogToChainsaw c = new SendLogToChainsaw(opts.dir, opts.filter, opts.hostname, opts.portnum, opts.startDate, opts.endDate, opts.regex, opts.level);
+ c.processLogFiles();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/TServerUtils.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/TServerUtils.java
index 50476a2,0000000..fa4de30
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/TServerUtils.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/TServerUtils.java
@@@ -1,313 -1,0 +1,314 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.UnknownHostException;
+import java.nio.channels.ServerSocketChannel;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.LoggingRunnable;
+import org.apache.accumulo.core.util.SimpleThreadPool;
+import org.apache.accumulo.core.util.TBufferedSocket;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.thrift.metrics.ThriftMetrics;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+import org.apache.thrift.TProcessor;
+import org.apache.thrift.TProcessorFactory;
+import org.apache.thrift.protocol.TProtocol;
+import org.apache.thrift.server.TServer;
+import org.apache.thrift.server.TThreadPoolServer;
+import org.apache.thrift.transport.TNonblockingSocket;
+import org.apache.thrift.transport.TServerTransport;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+
+public class TServerUtils {
+ private static final Logger log = Logger.getLogger(TServerUtils.class);
+
+ public static final ThreadLocal<String> clientAddress = new ThreadLocal<String>();
+
+ public static class ServerPort {
+ public final TServer server;
+ public final int port;
+
+ public ServerPort(TServer server, int port) {
+ this.server = server;
+ this.port = port;
+ }
+ }
+
+ /**
+ * Start a server, at the given port, or higher, if that port is not available.
+ *
+ * @param portHintProperty
+ * the port to attempt to open, can be zero, meaning "any available port"
+ * @param processor
+ * the service to be started
+ * @param serverName
+ * the name of the class that is providing the service
+ * @param threadName
+ * name this service's thread for better debugging
- * @param portSearchProperty
- * @param minThreadProperty
- * @param timeBetweenThreadChecksProperty
+ * @return the server object created, and the port actually used
+ * @throws UnknownHostException
+ * when we don't know our own address
+ */
+ public static ServerPort startServer(AccumuloConfiguration conf, Property portHintProperty, TProcessor processor, String serverName, String threadName,
+ Property portSearchProperty,
+ Property minThreadProperty,
+ Property timeBetweenThreadChecksProperty,
+ Property maxMessageSizeProperty) throws UnknownHostException {
+ int portHint = conf.getPort(portHintProperty);
+ int minThreads = 2;
+ if (minThreadProperty != null)
+ minThreads = conf.getCount(minThreadProperty);
+ long timeBetweenThreadChecks = 1000;
+ if (timeBetweenThreadChecksProperty != null)
+ timeBetweenThreadChecks = conf.getTimeInMillis(timeBetweenThreadChecksProperty);
+ long maxMessageSize = 10 * 1000 * 1000;
+ if (maxMessageSizeProperty != null)
+ maxMessageSize = conf.getMemoryInBytes(maxMessageSizeProperty);
+ boolean portSearch = false;
+ if (portSearchProperty != null)
+ portSearch = conf.getBoolean(portSearchProperty);
+ Random random = new Random();
+ for (int j = 0; j < 100; j++) {
+
+ // Are we going to slide around, looking for an open port?
+ int portsToSearch = 1;
+ if (portSearch)
+ portsToSearch = 1000;
+
+ for (int i = 0; i < portsToSearch; i++) {
+ int port = portHint + i;
+ if (portHint != 0 && i > 0)
+ port = 1024 + random.nextInt(65535 - 1024);
+ if (port > 65535)
+ port = 1024 + port % (65535 - 1024);
+ try {
+ return TServerUtils.startTServer(port, processor, serverName, threadName, minThreads, timeBetweenThreadChecks, maxMessageSize);
+ } catch (Exception ex) {
+ log.info("Unable to use port " + port + ", retrying. (Thread Name = " + threadName + ")");
+ UtilWaitThread.sleep(250);
+ }
+ }
+ }
+ throw new UnknownHostException("Unable to find a listen port");
+ }
+
+ public static class TimedProcessor implements TProcessor {
+
+ final TProcessor other;
+ ThriftMetrics metrics = null;
+ long idleStart = 0;
+
+ TimedProcessor(TProcessor next, String serverName, String threadName) {
+ this.other = next;
+ // Register the metrics MBean
+ try {
+ metrics = new ThriftMetrics(serverName, threadName);
+ metrics.register();
+ } catch (Exception e) {
+ log.error("Exception registering MBean with MBean Server", e);
+ }
+ idleStart = System.currentTimeMillis();
+ }
+
+ @Override
+ public boolean process(TProtocol in, TProtocol out) throws TException {
+ long now = 0;
+ if (metrics.isEnabled()) {
+ now = System.currentTimeMillis();
+ metrics.add(ThriftMetrics.idle, (now - idleStart));
+ }
+ try {
+ try {
+ return other.process(in, out);
+ } catch (NullPointerException ex) {
+ // THRIFT-1447 - remove with thrift 0.9
+ return true;
+ }
+ } finally {
+ if (metrics.isEnabled()) {
+ idleStart = System.currentTimeMillis();
+ metrics.add(ThriftMetrics.execute, idleStart - now);
+ }
+ }
+ }
+ }
+
+ public static class ClientInfoProcessorFactory extends TProcessorFactory {
+
+ public ClientInfoProcessorFactory(TProcessor processor) {
+ super(processor);
+ }
+
++ @Override
+ public TProcessor getProcessor(TTransport trans) {
+ if (trans instanceof TBufferedSocket) {
+ TBufferedSocket tsock = (TBufferedSocket) trans;
+ clientAddress.set(tsock.getClientString());
+ }
+ return super.getProcessor(trans);
+ }
+ }
+
+ public static class THsHaServer extends org.apache.thrift.server.THsHaServer {
+ public THsHaServer(Args args) {
+ super(args);
+ }
+
++ @Override
+ protected Runnable getRunnable(FrameBuffer frameBuffer) {
+ return new Invocation(frameBuffer);
+ }
+
+ private class Invocation implements Runnable {
+
+ private final FrameBuffer frameBuffer;
+
+ public Invocation(final FrameBuffer frameBuffer) {
+ this.frameBuffer = frameBuffer;
+ }
+
++ @Override
+ public void run() {
+ if (frameBuffer.trans_ instanceof TNonblockingSocket) {
+ TNonblockingSocket tsock = (TNonblockingSocket) frameBuffer.trans_;
+ Socket sock = tsock.getSocketChannel().socket();
+ clientAddress.set(sock.getInetAddress().getHostAddress() + ":" + sock.getPort());
+ }
+ frameBuffer.invoke();
+ }
+ }
+ }
+
+ public static ServerPort startHsHaServer(int port, TProcessor processor, final String serverName, String threadName, final int numThreads,
+ long timeBetweenThreadChecks, long maxMessageSize) throws TTransportException {
+ TNonblockingServerSocket transport = new TNonblockingServerSocket(port);
+ if (port == 0) {
+ port = transport.getPort();
+ }
+ THsHaServer.Args options = new THsHaServer.Args(transport);
+ options.protocolFactory(ThriftUtil.protocolFactory());
+ options.transportFactory(ThriftUtil.transportFactory(maxMessageSize));
+ options.maxReadBufferBytes = maxMessageSize;
+ options.stopTimeoutVal(5);
+ /*
+ * Create our own very special thread pool.
+ */
+ final ThreadPoolExecutor pool = new SimpleThreadPool(numThreads, "ClientPool");
+ // periodically adjust the number of threads we need by checking how busy our threads are
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ if (pool.getCorePoolSize() <= pool.getActiveCount()) {
+ int larger = pool.getCorePoolSize() + Math.min(pool.getQueue().size(), 2);
+ log.info("Increasing server thread pool size on " + serverName + " to " + larger);
+ pool.setMaximumPoolSize(larger);
+ pool.setCorePoolSize(larger);
+ } else {
+ if (pool.getCorePoolSize() > pool.getActiveCount() + 3) {
+ int smaller = Math.max(numThreads, pool.getCorePoolSize() - 1);
+ if (smaller != pool.getCorePoolSize()) {
+ // there is a race condition here... the active count could be higher by the time
+ // we decrease the core pool size... so the active count could end up higher than
+ // the core pool size, in which case everything will be queued... the increase case
+ // should handle this and prevent deadlock
+ log.info("Decreasing server thread pool size on " + serverName + " to " + smaller);
+ pool.setCorePoolSize(smaller);
+ }
+ }
+ }
+ }
+ }, timeBetweenThreadChecks, timeBetweenThreadChecks);
+ options.executorService(pool);
+ processor = new TServerUtils.TimedProcessor(processor, serverName, threadName);
+ options.processorFactory(new TProcessorFactory(processor));
+ return new ServerPort(new THsHaServer(options), port);
+ }
+
+ public static ServerPort startThreadPoolServer(int port, TProcessor processor, String serverName, String threadName, int numThreads)
+ throws TTransportException {
+
+ // if port is zero, then we must bind to get the port number
+ ServerSocket sock;
+ try {
+ sock = ServerSocketChannel.open().socket();
+ sock.setReuseAddress(true);
+ sock.bind(new InetSocketAddress(port));
+ port = sock.getLocalPort();
+ } catch (IOException ex) {
+ throw new TTransportException(ex);
+ }
+ TServerTransport transport = new TBufferedServerSocket(sock, 32 * 1024);
+ TThreadPoolServer.Args options = new TThreadPoolServer.Args(transport);
+ options.protocolFactory(ThriftUtil.protocolFactory());
+ options.transportFactory(ThriftUtil.transportFactory());
+ processor = new TServerUtils.TimedProcessor(processor, serverName, threadName);
+ options.processorFactory(new ClientInfoProcessorFactory(processor));
+ return new ServerPort(new TThreadPoolServer(options), port);
+ }
+
+ public static ServerPort startTServer(int port, TProcessor processor, String serverName, String threadName, int numThreads, long timeBetweenThreadChecks, long maxMessageSize)
+ throws TTransportException {
+ ServerPort result = startHsHaServer(port, processor, serverName, threadName, numThreads, timeBetweenThreadChecks, maxMessageSize);
+ // ServerPort result = startThreadPoolServer(port, processor, serverName, threadName, -1);
+ final TServer finalServer = result.server;
+ Runnable serveTask = new Runnable() {
++ @Override
+ public void run() {
+ try {
+ finalServer.serve();
+ } catch (Error e) {
+ Halt.halt("Unexpected error in TThreadPoolServer " + e + ", halting.");
+ }
+ }
+ };
+ serveTask = new LoggingRunnable(TServerUtils.log, serveTask);
+ Thread thread = new Daemon(serveTask, threadName);
+ thread.start();
+ return result;
+ }
+
+ // Existing connections will keep our thread running: reach in with reflection and insist that they shutdown.
+ public static void stopTServer(TServer s) {
+ if (s == null)
+ return;
+ s.stop();
+ try {
+ Field f = s.getClass().getDeclaredField("executorService_");
+ f.setAccessible(true);
+ ExecutorService es = (ExecutorService) f.get(s);
+ es.shutdownNow();
+ } catch (Exception e) {
+ TServerUtils.log.error("Unable to call shutdownNow", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
index c3f4a72,0000000..92e0674
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
@@@ -1,48 -1,0 +1,45 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+
+import com.beust.jcommander.Parameter;
+
+public class TableDiskUsage {
+
+ static class Opts extends ClientOpts {
+ @Parameter(description=" <table> { <table> ... } ")
+ List<String> tables = new ArrayList<String>();
+ }
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ FileSystem fs = FileSystem.get(new Configuration());
+ Opts opts = new Opts();
+ opts.parseArgs(TableDiskUsage.class.getName(), args);
+ Connector conn = opts.getConnector();
+ org.apache.accumulo.core.util.TableDiskUsage.printDiskUsage(DefaultConfiguration.getInstance(), opts.tables, fs, conn, false);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
index 2fc0bd3,0000000..34c2151
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
@@@ -1,75 -1,0 +1,72 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.util.List;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+
+import com.beust.jcommander.Parameter;
+
+public class TabletServerLocks {
+
+ static class Opts extends Help {
+ @Parameter(names="-list")
+ boolean list = false;
+ @Parameter(names="-delete")
+ String delete = null;
+ }
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+
+ Instance instance = HdfsZooInstance.getInstance();
+ String tserverPath = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
+ Opts opts = new Opts();
+ opts.parseArgs(TabletServerLocks.class.getName(), args);
+
+ ZooCache cache = new ZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
+
+ if (opts.list) {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ List<String> tabletServers = zoo.getChildren(tserverPath);
+
+ for (String tabletServer : tabletServers) {
+ byte[] lockData = ZooLock.getLockData(cache, tserverPath + "/" + tabletServer, null);
+ String holder = null;
+ if (lockData != null) {
+ holder = new String(lockData, Constants.UTF8);
+ }
+
+ System.out.printf("%32s %16s%n", tabletServer, holder);
+ }
+ } else if (opts.delete != null) {
+ ZooLock.deleteLock(tserverPath + "/" + args[1]);
+ } else {
+ System.out.println("Usage : " + TabletServerLocks.class.getName() + " -list|-delete <tserver lock>");
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
----------------------------------------------------------------------
diff --cc start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
index 5f7fd5e,0000000..1e2b4b5
mode 100644,000000..100644
--- a/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
+++ b/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
@@@ -1,255 -1,0 +1,252 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.start.classloader;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import org.apache.log4j.Logger;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+/**
+ *
+ */
+public class AccumuloClassLoader {
+
+ public static final String CLASSPATH_PROPERTY_NAME = "general.classpaths";
+
+ public static final String ACCUMULO_CLASSPATH_VALUE =
+ "$ACCUMULO_CONF_DIR,\n" +
+ "$ACCUMULO_HOME/lib/[^.].*.jar,\n" +
+ "$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n" +
+ "$HADOOP_CONF_DIR,\n" +
+ "$HADOOP_PREFIX/[^.].*.jar,\n" +
+ "$HADOOP_PREFIX/lib/[^.].*.jar,\n" +
+ "$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n" +
+ "$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n" +
+ "$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n" +
+ "$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n" +
+ "/usr/lib/hadoop/[^.].*.jar,\n" +
+ "/usr/lib/hadoop/lib/[^.].*.jar,\n" +
+ "/usr/lib/hadoop-hdfs/[^.].*.jar,\n" +
+ "/usr/lib/hadoop-mapreduce/[^.].*.jar,\n" +
+ "/usr/lib/hadoop-yarn/[^.].*.jar,\n"
+ ;
+
+ private static String SITE_CONF;
+
+ private static URLClassLoader classloader;
+
+ private static Logger log = Logger.getLogger(AccumuloClassLoader.class);
+
+ static {
+ String configFile = System.getProperty("org.apache.accumulo.config.file", "accumulo-site.xml");
+ if (System.getenv("ACCUMULO_CONF_DIR") != null) {
+ // accumulo conf dir should be set
+ SITE_CONF = System.getenv("ACCUMULO_CONF_DIR") + "/" + configFile;
+ } else if (System.getenv("ACCUMULO_HOME") != null) {
+ // if no accumulo conf dir, try accumulo home default
+ SITE_CONF = System.getenv("ACCUMULO_HOME") + "/conf/" + configFile;
+ } else {
+ SITE_CONF = null;
+ }
+ }
+
+ /**
+ * Parses and XML Document for a property node for a <name> with the value propertyName if it finds one the function return that property's value for its
+ * <value> node. If not found the function will return null
+ *
+ * @param d
+ * XMLDocument to search through
+ * @param propertyName
+ */
+ private static String getAccumuloClassPathStrings(Document d, String propertyName) {
+ NodeList pnodes = d.getElementsByTagName("property");
+ for (int i = pnodes.getLength() - 1; i >= 0; i--) {
+ Element current_property = (Element) pnodes.item(i);
+ Node cname = current_property.getElementsByTagName("name").item(0);
+ if (cname != null && cname.getTextContent().compareTo(propertyName) == 0) {
+ Node cvalue = current_property.getElementsByTagName("value").item(0);
+ if (cvalue != null) {
+ return cvalue.getTextContent();
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Looks for the site configuration file for Accumulo and if it has a property for propertyName return it otherwise returns defaultValue Should throw an
+ * exception if the default configuration can not be read;
+ *
+ * @param propertyName
+ * Name of the property to pull
+ * @param defaultValue
+ * Value to default to if not found.
+ * @return site or default class path String
+ */
+
+ public static String getAccumuloString(String propertyName, String defaultValue) {
+
+ try {
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ String site_classpath_string = null;
+ try {
+ Document site_conf = db.parse(SITE_CONF);
+ site_classpath_string = getAccumuloClassPathStrings(site_conf, propertyName);
+ } catch (Exception e) {
+ /* we don't care because this is optional and we can use defaults */
+ }
+ if (site_classpath_string != null)
+ return site_classpath_string;
+ return defaultValue;
+ } catch (Exception e) {
+ throw new IllegalStateException("ClassPath Strings Lookup failed", e);
+ }
+ }
+
+ /**
+ * Replace environment variables in the classpath string with their actual value
- *
- * @param classpath
- * @param env
+ */
+ public static String replaceEnvVars(String classpath, Map<String,String> env) {
+ Pattern envPat = Pattern.compile("\\$[A-Za-z][a-zA-Z0-9_]*");
+ Matcher envMatcher = envPat.matcher(classpath);
+ while (envMatcher.find(0)) {
+ // name comes after the '$'
+ String varName = envMatcher.group().substring(1);
+ String varValue = env.get(varName);
+ if (varValue == null) {
+ varValue = "";
+ }
+ classpath = (classpath.substring(0, envMatcher.start()) + varValue + classpath.substring(envMatcher.end()));
+ envMatcher.reset(classpath);
+ }
+ return classpath;
+ }
+
+ /**
+ * Populate the list of URLs with the items in the classpath string
+ *
+ * @param classpath
+ * @param urls
+ * @throws MalformedURLException
+ */
+ private static void addUrl(String classpath, ArrayList<URL> urls) throws MalformedURLException {
+ classpath = classpath.trim();
+ if (classpath.length() == 0)
+ return;
+
+ classpath = replaceEnvVars(classpath, System.getenv());
+
+ // Try to make a URI out of the classpath
+ URI uri = null;
+ try {
+ uri = new URI(classpath);
+ } catch (URISyntaxException e) {
+ // Not a valid URI
+ }
+
+ if (null == uri || !uri.isAbsolute() || (null != uri.getScheme() && uri.getScheme().equals("file://"))) {
+ // Then treat this URI as a File.
+ // This checks to see if the url string is a dir if it expand and get all jars in that directory
+ final File extDir = new File(classpath);
+ if (extDir.isDirectory())
+ urls.add(extDir.toURI().toURL());
+ else {
+ if (extDir.getParentFile() != null) {
+ File[] extJars = extDir.getParentFile().listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return name.matches("^" + extDir.getName());
+ }
+ });
+ if (extJars != null && extJars.length > 0) {
+ for (File jar : extJars)
+ urls.add(jar.toURI().toURL());
+ } else {
+ log.debug("ignoring classpath entry " + classpath);
+ }
+ } else {
+ log.debug("ignoring classpath entry " + classpath);
+ }
+ }
+ } else {
+ urls.add(uri.toURL());
+ }
+
+ }
+
+ private static ArrayList<URL> findAccumuloURLs() throws IOException {
+ String cp = getAccumuloString(AccumuloClassLoader.CLASSPATH_PROPERTY_NAME, AccumuloClassLoader.ACCUMULO_CLASSPATH_VALUE);
+ if (cp == null)
+ return new ArrayList<URL>();
+ String[] cps = replaceEnvVars(cp, System.getenv()).split(",");
+ ArrayList<URL> urls = new ArrayList<URL>();
+ for (String classpath : cps) {
+ if (!classpath.startsWith("#")) {
+ addUrl(classpath, urls);
+ }
+ }
+ return urls;
+ }
+
+ public static synchronized ClassLoader getClassLoader() throws IOException {
+ if (classloader == null) {
+ ArrayList<URL> urls = findAccumuloURLs();
+
+ ClassLoader parentClassLoader = AccumuloClassLoader.class.getClassLoader();
+
+ log.debug("Create 2nd tier ClassLoader using URLs: " + urls.toString());
+ URLClassLoader aClassLoader = new URLClassLoader(urls.toArray(new URL[urls.size()]), parentClassLoader) {
+ @Override
+ protected synchronized Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
+
+ if (name.startsWith("org.apache.accumulo.start.classloader.vfs")) {
+ Class<?> c = findLoadedClass(name);
+ if (c == null) {
+ try {
+ // try finding this class here instead of parent
+ findClass(name);
+ } catch (ClassNotFoundException e) {}
+ }
+ }
+ return super.loadClass(name, resolve);
+ }
+ };
+ classloader = aClassLoader;
+ }
+
+ return classloader;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
----------------------------------------------------------------------
diff --cc start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
index 7742cbe,0000000..e1ff55e
mode 100644,000000..100644
--- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
+++ b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
@@@ -1,207 -1,0 +1,205 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.start.classloader.vfs;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileSystemManager;
+
+public class ContextManager {
+
+ // there is a lock per context so that one context can initialize w/o blocking another context
+ private class Context {
+ AccumuloReloadingVFSClassLoader loader;
+ ContextConfig cconfig;
+ boolean closed = false;
+
+ Context(ContextConfig cconfig) {
+ this.cconfig = cconfig;
+ }
+
+ synchronized ClassLoader getClassLoader() throws FileSystemException {
+ if (closed)
+ return null;
+
+ if (loader == null) {
+ loader = new AccumuloReloadingVFSClassLoader(cconfig.uris, vfs, parent, cconfig.preDelegation);
+ }
+
+ return loader.getClassLoader();
+ }
+
+ synchronized void close() {
+ closed = true;
+ loader.close();
+ loader = null;
+ }
+ }
+
+ private Map<String,Context> contexts = new HashMap<String,Context>();
+
+ private volatile ContextsConfig config;
+ private FileSystemManager vfs;
+ private ReloadingClassLoader parent;
+
+ ContextManager(FileSystemManager vfs, ReloadingClassLoader parent) {
+ this.vfs = vfs;
+ this.parent = parent;
+ }
+
+ public static class ContextConfig {
+ String uris;
+ boolean preDelegation;
+
+ public ContextConfig(String uris, boolean preDelegation) {
+ this.uris = uris;
+ this.preDelegation = preDelegation;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof ContextConfig) {
+ ContextConfig oc = (ContextConfig) o;
+
+ return uris.equals(oc.uris) && preDelegation == oc.preDelegation;
+ }
+
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return uris.hashCode() + (preDelegation ? Boolean.TRUE : Boolean.FALSE).hashCode();
+ }
+ }
+
+ public interface ContextsConfig {
+ ContextConfig getContextConfig(String context);
+ }
+
+ public static class DefaultContextsConfig implements ContextsConfig {
+
+ private Iterable<Entry<String,String>> config;
+
+ public DefaultContextsConfig(Iterable<Entry<String,String>> config) {
+ this.config = config;
+ }
+
+ @Override
+ public ContextConfig getContextConfig(String context) {
+
+ String key = AccumuloVFSClassLoader.VFS_CONTEXT_CLASSPATH_PROPERTY + context;
+
+ String uris = null;
+ boolean preDelegate = true;
+
+ Iterator<Entry<String,String>> iter = config.iterator();
+ while (iter.hasNext()) {
+ Entry<String,String> entry = iter.next();
+ if (entry.getKey().equals(key)) {
+ uris = entry.getValue();
+ }
+
+ if (entry.getKey().equals(key + ".delegation") && entry.getValue().trim().equalsIgnoreCase("post")) {
+ preDelegate = false;
+ }
+ }
+
+ if (uris != null)
+ return new ContextConfig(uris, preDelegate);
+
+ return null;
+ }
+ }
+
+ /**
+ * configuration must be injected for ContextManager to work
- *
- * @param config
+ */
+ public synchronized void setContextConfig(ContextsConfig config) {
+ if (this.config != null)
+ throw new IllegalStateException("Context manager config already set");
+ this.config = config;
+ }
+
+ public ClassLoader getClassLoader(String contextName) throws FileSystemException {
+
+ ContextConfig cconfig = config.getContextConfig(contextName);
+
+ if (cconfig == null)
+ throw new IllegalArgumentException("Unknown context " + contextName);
+
+ Context context = null;
+ Context contextToClose = null;
+
+ synchronized (this) {
+ // only manipulate internal data structs in this sync block... avoid creating or closing classloader, reading config, etc... basically avoid operations
+ // that may block
+ context = contexts.get(contextName);
+
+ if (context == null) {
+ context = new Context(cconfig);
+ contexts.put(contextName, context);
+ } else if (!context.cconfig.equals(cconfig)) {
+ contextToClose = context;
+ context = new Context(cconfig);
+ contexts.put(contextName, context);
+ }
+ }
+
+ if (contextToClose != null)
+ contextToClose.close();
+
+ ClassLoader loader = context.getClassLoader();
+ if (loader == null) {
+ // ooppss, context was closed by another thread, try again
+ return getClassLoader(contextName);
+ }
+
+ return loader;
+
+ }
+
+ public <U> Class<? extends U> loadClass(String context, String classname, Class<U> extension) throws ClassNotFoundException {
+ try {
+ return getClassLoader(context).loadClass(classname).asSubclass(extension);
+ } catch (IOException e) {
+ throw new ClassNotFoundException("IO Error loading class " + classname, e);
+ }
+ }
+
+ public void removeUnusedContexts(Set<String> inUse) {
+
+ Map<String,Context> unused;
+
+ synchronized (this) {
+ unused = new HashMap<String,Context>(contexts);
+ unused.keySet().removeAll(inUse);
+ contexts.keySet().removeAll(unused.keySet());
+ }
+
+ for (Context context : unused.values()) {
+ // close outside of lock
+ context.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
----------------------------------------------------------------------
diff --cc start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
index 0a6931f,0000000..277c741
mode 100644,000000..100644
--- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
+++ b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
@@@ -1,52 -1,0 +1,47 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.start.classloader.vfs;
+
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileSystemManager;
+import org.apache.commons.vfs2.impl.VFSClassLoader;
+
+/**
+ *
+ */
+public class PostDelegatingVFSClassLoader extends VFSClassLoader {
+
- /**
- * @param files
- * @param manager
- * @param parent
- * @throws FileSystemException
- */
+ public PostDelegatingVFSClassLoader(FileObject[] files, FileSystemManager manager, ClassLoader parent) throws FileSystemException {
+ super(files, manager, parent);
+ }
+
++ @Override
+ protected synchronized Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
+ Class<?> c = findLoadedClass(name);
+ if (c == null) {
+ try {
+ // try finding this class here instead of parent
+ findClass(name);
+ } catch (ClassNotFoundException e) {
+
+ }
+ }
+ return super.loadClass(name, resolve);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java
----------------------------------------------------------------------
diff --cc start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java
index 92b2720,0000000..104ea09
mode 100644,000000..100644
--- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java
+++ b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java
@@@ -1,164 -1,0 +1,159 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.start.classloader.vfs.providers;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.vfs2.CacheStrategy;
+import org.apache.commons.vfs2.Capability;
+import org.apache.commons.vfs2.FileName;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileSystemOptions;
+import org.apache.commons.vfs2.provider.AbstractFileName;
+import org.apache.commons.vfs2.provider.AbstractFileSystem;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * A VFS FileSystem that interacts with HDFS.
+ *
+ * @since 2.1
+ */
+public class HdfsFileSystem extends AbstractFileSystem
+{
+ private static final Log log = LogFactory.getLog(HdfsFileSystem.class);
+
+ private FileSystem fs;
+
- /**
- *
- * @param rootName
- * @param fileSystemOptions
- */
+ protected HdfsFileSystem(final FileName rootName, final FileSystemOptions fileSystemOptions)
+ {
+ super(rootName, null, fileSystemOptions);
+ }
+
+ /**
+ * @see org.apache.commons.vfs2.provider.AbstractFileSystem#addCapabilities(java.util.Collection)
+ */
+ @Override
+ protected void addCapabilities(final Collection<Capability> capabilities)
+ {
+ capabilities.addAll(HdfsFileProvider.CAPABILITIES);
+ }
+
+ /**
+ * @see org.apache.commons.vfs2.provider.AbstractFileSystem#close()
+ */
+ @Override
+ public void close()
+ {
+ try
+ {
+ if (null != fs)
+ {
+ fs.close();
+ }
+ }
+ catch (final IOException e)
+ {
+ throw new RuntimeException("Error closing HDFS client", e);
+ }
+ super.close();
+ }
+
+ /**
+ * @see org.apache.commons.vfs2.provider.AbstractFileSystem#createFile(org.apache.commons.vfs2.provider.AbstractFileName)
+ */
+ @Override
+ protected FileObject createFile(final AbstractFileName name) throws Exception
+ {
+ throw new FileSystemException("Operation not supported");
+ }
+
+ /**
+ * @see org.apache.commons.vfs2.provider.AbstractFileSystem#resolveFile(org.apache.commons.vfs2.FileName)
+ */
+ @Override
+ public FileObject resolveFile(final FileName name) throws FileSystemException
+ {
+
+ synchronized (this)
+ {
+ if (null == this.fs)
+ {
+ final String hdfsUri = name.getRootURI();
+ final Configuration conf = new Configuration(true);
+ conf.set(org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY, hdfsUri);
+ this.fs = null;
+ try
+ {
+ fs = org.apache.hadoop.fs.FileSystem.get(conf);
+ }
+ catch (final IOException e)
+ {
+ log.error("Error connecting to filesystem " + hdfsUri, e);
+ throw new FileSystemException("Error connecting to filesystem " + hdfsUri, e);
+ }
+ }
+ }
+
+ boolean useCache = (null != getContext().getFileSystemManager().getFilesCache());
+ FileObject file;
+ if (useCache)
+ {
+ file = this.getFileFromCache(name);
+ }
+ else
+ {
+ file = null;
+ }
+ if (null == file)
+ {
+ String path = null;
+ try
+ {
+ path = URLDecoder.decode(name.getPath(), "UTF-8");
+ }
+ catch (final UnsupportedEncodingException e)
+ {
+ path = name.getPath();
+ }
+ final Path filePath = new Path(path);
+ file = new HdfsFileObject((AbstractFileName) name, this, fs, filePath);
+ if (useCache)
+ {
+ this.putFileToCache(file);
+ }
+
+ }
+
+ /**
+ * resync the file information if requested
+ */
+ if (getFileSystemManager().getCacheStrategy().equals(CacheStrategy.ON_RESOLVE)) {
+ file.refresh();
+ }
+
+ return file;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/GetMasterStats.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/GetMasterStats.java
index 65cf80c,0000000..7b1313a
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/GetMasterStats.java
+++ b/test/src/main/java/org/apache/accumulo/test/GetMasterStats.java
@@@ -1,129 -1,0 +1,120 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
- import java.io.IOException;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.accumulo.core.client.impl.MasterClient;
- import org.apache.accumulo.core.master.MasterNotRunningException;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
+import org.apache.accumulo.core.master.thrift.RecoveryStatus;
+import org.apache.accumulo.core.master.thrift.TableInfo;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.monitor.Monitor;
+import org.apache.accumulo.server.security.SecurityConstants;
- import org.apache.thrift.transport.TTransportException;
+
+public class GetMasterStats {
- /**
- * @param args
- * @throws MasterNotRunningException
- * @throws IOException
- * @throws TTransportException
- */
+ public static void main(String[] args) throws Exception {
+ MasterClientService.Iface client = null;
+ MasterMonitorInfo stats = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(HdfsZooInstance.getInstance());
+ stats = client.getMasterStats(Tracer.traceInfo(), SecurityConstants.getSystemCredentials());
+ } finally {
+ if (client != null)
+ MasterClient.close(client);
+ }
+ out(0, "State: " + stats.state.name());
+ out(0, "Goal State: " + stats.goalState.name());
+ if (stats.serversShuttingDown != null && stats.serversShuttingDown.size() > 0) {
+ out(0, "Servers to shutdown");
+ for (String server : stats.serversShuttingDown) {
+ out(1, "%s", server);
+ }
+ }
+ out(0, "Unassigned tablets: %d", stats.unassignedTablets);
+ if (stats.badTServers != null && stats.badTServers.size() > 0) {
+ out(0, "Bad servers");
+
+ for (Entry<String,Byte> entry : stats.badTServers.entrySet()) {
+ out(1, "%s: %d", entry.getKey(), (int) entry.getValue());
+ }
+ }
+ if (stats.tableMap != null && stats.tableMap.size() > 0) {
+ out(0, "Tables");
+ for (Entry<String,TableInfo> entry : stats.tableMap.entrySet()) {
+ TableInfo v = entry.getValue();
+ out(1, "%s", entry.getKey());
+ out(2, "Records: %d", v.recs);
+ out(2, "Records in Memory: %d", v.recsInMemory);
+ out(2, "Tablets: %d", v.tablets);
+ out(2, "Online Tablets: %d", v.onlineTablets);
+ out(2, "Ingest Rate: %.2f", v.ingestRate);
+ out(2, "Query Rate: %.2f", v.queryRate);
+ }
+ }
+ if (stats.tServerInfo != null && stats.tServerInfo.size() > 0) {
+ out(0, "Tablet Servers");
+ long now = System.currentTimeMillis();
+ for (TabletServerStatus server : stats.tServerInfo) {
+ TableInfo summary = Monitor.summarizeTableStats(server);
+ out(1, "Name: %s", server.name);
+ out(2, "Ingest: %.2f", summary.ingestRate);
+ out(2, "Last Contact: %s", server.lastContact);
+ out(2, "OS Load Average: %.2f", server.osLoad);
+ out(2, "Queries: %.2f", summary.queryRate);
+ out(2, "Time Difference: %.1f", ((now - server.lastContact) / 1000.));
+ out(2, "Total Records: %d", summary.recs);
+ out(2, "Lookups: %d", server.lookups);
+ if (server.holdTime > 0)
+ out(2, "Hold Time: %d", server.holdTime);
+ if (server.tableMap != null && server.tableMap.size() > 0) {
+ out(2, "Tables");
+ for (Entry<String,TableInfo> status : server.tableMap.entrySet()) {
+ TableInfo info = status.getValue();
+ out(3, "Table: %s", status.getKey());
+ out(4, "Tablets: %d", info.onlineTablets);
+ out(4, "Records: %d", info.recs);
+ out(4, "Records in Memory: %d", info.recsInMemory);
+ out(4, "Ingest: %.2f", info.ingestRate);
+ out(4, "Queries: %.2f", info.queryRate);
+ out(4, "Major Compacting: %d", info.majors == null ? 0 : info.majors.running);
+ out(4, "Queued for Major Compaction: %d", info.majors == null ? 0 : info.majors.queued);
+ out(4, "Minor Compacting: %d", info.minors == null ? 0 : info.minors.running);
+ out(4, "Queued for Minor Compaction: %d", info.minors == null ? 0 : info.minors.queued);
+ }
+ }
+ out(2, "Recoveries: %d", server.logSorts.size());
+ for (RecoveryStatus sort : server.logSorts) {
+ out(3, "File: %s", sort.name);
+ out(3, "Progress: %.2f%%", sort.progress * 100);
+ out(3, "Time running: %s", sort.runtime / 1000.);
+ }
+ }
+ }
+ }
+
+ private static void out(int indent, String string, Object... args) {
+ for (int i = 0; i < indent; i++) {
+ System.out.print(" ");
+ }
+ System.out.println(String.format(string, args));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
index 73b73f4,0000000..16e7a98
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
+++ b/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
@@@ -1,198 -1,0 +1,195 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.FastFormat;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.tabletserver.NativeMap;
+import org.apache.hadoop.io.Text;
+
+public class NativeMapPerformanceTest {
+
+ private static final byte ROW_PREFIX[] = new byte[] {'r'};
+ private static final byte COL_PREFIX[] = new byte[] {'c'};
+
+ static Key nk(int r, int c) {
+ return new Key(new Text(FastFormat.toZeroPaddedString(r, 9, 10, ROW_PREFIX)), new Text(FastFormat.toZeroPaddedString(c, 6, 10, COL_PREFIX)));
+ }
+
+ static Mutation nm(int r) {
+ return new Mutation(new Text(FastFormat.toZeroPaddedString(r, 9, 10, ROW_PREFIX)));
+ }
+
+ static Text ET = new Text();
+
+ private static void pc(Mutation m, int c, Value v) {
+ m.put(new Text(FastFormat.toZeroPaddedString(c, 6, 10, COL_PREFIX)), ET, Long.MAX_VALUE, v);
+ }
+
+ static void runPerformanceTest(int numRows, int numCols, int numLookups, String mapType) {
+
+ SortedMap<Key,Value> tm = null;
+ NativeMap nm = null;
+
+ if (mapType.equals("SKIP_LIST"))
+ tm = new ConcurrentSkipListMap<Key,Value>();
+ else if (mapType.equals("TREE_MAP"))
+ tm = Collections.synchronizedSortedMap(new TreeMap<Key,Value>());
+ else if (mapType.equals("NATIVE_MAP"))
+ nm = new NativeMap();
+ else
+ throw new IllegalArgumentException(" map type must be SKIP_LIST, TREE_MAP, or NATIVE_MAP");
+
+ Random rand = new Random(19);
+
+ // puts
+ long tps = System.currentTimeMillis();
+
+ if (nm != null) {
+ for (int i = 0; i < numRows; i++) {
+ int row = rand.nextInt(1000000000);
+ Mutation m = nm(row);
+ for (int j = 0; j < numCols; j++) {
+ int col = rand.nextInt(1000000);
+ Value val = new Value("test".getBytes(Constants.UTF8));
+ pc(m, col, val);
+ }
+ nm.mutate(m, i);
+ }
+ } else {
+ for (int i = 0; i < numRows; i++) {
+ int row = rand.nextInt(1000000000);
+ for (int j = 0; j < numCols; j++) {
+ int col = rand.nextInt(1000000);
+ Key key = nk(row, col);
+ Value val = new Value("test".getBytes(Constants.UTF8));
+ tm.put(key, val);
+ }
+ }
+ }
+
+ long tpe = System.currentTimeMillis();
+
+ // Iteration
+ Iterator<Entry<Key,Value>> iter;
+ if (nm != null) {
+ iter = nm.iterator();
+ } else {
+ iter = tm.entrySet().iterator();
+ }
+
+ long tis = System.currentTimeMillis();
+
+ while (iter.hasNext()) {
+ iter.next();
+ }
+
+ long tie = System.currentTimeMillis();
+
+ rand = new Random(19);
+ int rowsToLookup[] = new int[numLookups];
+ int colsToLookup[] = new int[numLookups];
+ for (int i = 0; i < Math.min(numLookups, numRows); i++) {
+ int row = rand.nextInt(1000000000);
+ int col = -1;
+ for (int j = 0; j < numCols; j++) {
+ col = rand.nextInt(1000000);
+ }
+
+ rowsToLookup[i] = row;
+ colsToLookup[i] = col;
+ }
+
+ // get
+
+ long tgs = System.currentTimeMillis();
+ if (nm != null) {
+ for (int i = 0; i < numLookups; i++) {
+ Key key = nk(rowsToLookup[i], colsToLookup[i]);
+ if (nm.get(key) == null) {
+ throw new RuntimeException("Did not find " + rowsToLookup[i] + " " + colsToLookup[i] + " " + i);
+ }
+ }
+ } else {
+ for (int i = 0; i < numLookups; i++) {
+ Key key = nk(rowsToLookup[i], colsToLookup[i]);
+ if (tm.get(key) == null) {
+ throw new RuntimeException("Did not find " + rowsToLookup[i] + " " + colsToLookup[i] + " " + i);
+ }
+ }
+ }
+ long tge = System.currentTimeMillis();
+
+ long memUsed = 0;
+ if (nm != null) {
+ memUsed = nm.getMemoryUsed();
+ }
+
+ int size = (nm == null ? tm.size() : nm.size());
+
+ // delete
+ long tds = System.currentTimeMillis();
+
+ if (nm != null)
+ nm.delete();
+
+ long tde = System.currentTimeMillis();
+
+ if (tm != null)
+ tm.clear();
+
+ System.gc();
+ System.gc();
+ System.gc();
+ System.gc();
+
+ UtilWaitThread.sleep(3000);
+
+ System.out.printf("mapType:%10s put rate:%,6.2f scan rate:%,6.2f get rate:%,6.2f delete time : %6.2f mem : %,d%n", "" + mapType, (numRows * numCols)
+ / ((tpe - tps) / 1000.0), (size) / ((tie - tis) / 1000.0), numLookups / ((tge - tgs) / 1000.0), (tde - tds) / 1000.0, memUsed);
+
+ }
+
- /**
- * @param args
- */
+ public static void main(String[] args) {
+
+ if (args.length != 3) {
+ throw new IllegalArgumentException("Usage : " + NativeMapPerformanceTest.class.getName() + " <map type> <rows> <columns>");
+ }
+
+ String mapType = args[0];
+ int rows = Integer.parseInt(args[1]);
+ int cols = Integer.parseInt(args[2]);
+
+ runPerformanceTest(rows, cols, 10000, mapType);
+ runPerformanceTest(rows, cols, 10000, mapType);
+ runPerformanceTest(rows, cols, 10000, mapType);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
index 8411c86,0000000..5115541
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
+++ b/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
@@@ -1,277 -1,0 +1,274 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.OpTimer;
+import org.apache.accumulo.server.tabletserver.NativeMap;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+public class NativeMapStressTest {
+
+ private static final Logger log = Logger.getLogger(NativeMapStressTest.class);
+
- /**
- * @param args
- */
+ public static void main(String[] args) {
+ testLotsOfMapDeletes(true);
+ testLotsOfMapDeletes(false);
+ testLotsOfOverwrites();
+ testLotsOfGetsAndScans();
+ }
+
+ private static void put(NativeMap nm, String row, String val, int mc) {
+ Mutation m = new Mutation(new Text(row));
+ m.put(new Text(), new Text(), Long.MAX_VALUE, new Value(val.getBytes(Constants.UTF8)));
+ nm.mutate(m, mc);
+ }
+
+ private static void testLotsOfGetsAndScans() {
+
+ ArrayList<Thread> threads = new ArrayList<Thread>();
+
+ final int numThreads = 8;
+ final int totalGets = 100000000;
+ final int mapSizePerThread = (int) (4000000 / (double) numThreads);
+ final int getsPerThread = (int) (totalGets / (double) numThreads);
+
+ for (int tCount = 0; tCount < numThreads; tCount++) {
+ Runnable r = new Runnable() {
+ @Override
+ public void run() {
+ NativeMap nm = new NativeMap();
+
+ Random r = new Random();
+
+ OpTimer opTimer = new OpTimer(log, Level.INFO);
+
+ opTimer.start("Creating map of size " + mapSizePerThread);
+
+ for (int i = 0; i < mapSizePerThread; i++) {
+ String row = String.format("r%08d", i);
+ String val = row + "v";
+ put(nm, row, val, i);
+ }
+
+ opTimer.stop("Created map of size " + nm.size() + " in %DURATION%");
+
+ opTimer.start("Doing " + getsPerThread + " gets()");
+
+ for (int i = 0; i < getsPerThread; i++) {
+ String row = String.format("r%08d", r.nextInt(mapSizePerThread));
+ String val = row + "v";
+
+ Value value = nm.get(new Key(new Text(row)));
+ if (value == null || !value.toString().equals(val)) {
+ log.error("nm.get(" + row + ") failed");
+ }
+ }
+
+ opTimer.stop("Finished " + getsPerThread + " gets in %DURATION%");
+
+ int scanned = 0;
+
+ opTimer.start("Doing " + getsPerThread + " random iterations");
+
+ for (int i = 0; i < getsPerThread; i++) {
+ int startRow = r.nextInt(mapSizePerThread);
+ String row = String.format("r%08d", startRow);
+
+ Iterator<Entry<Key,Value>> iter = nm.iterator(new Key(new Text(row)));
+
+ int count = 0;
+
+ while (iter.hasNext() && count < 10) {
+ String row2 = String.format("r%08d", startRow + count);
+ String val2 = row2 + "v";
+
+ Entry<Key,Value> entry = iter.next();
+ if (!entry.getValue().toString().equals(val2) || !entry.getKey().equals(new Key(new Text(row2)))) {
+ log.error("nm.iter(" + row2 + ") failed row = " + row + " count = " + count + " row2 = " + row + " val2 = " + val2);
+ }
+
+ count++;
+ }
+
+ scanned += count;
+ }
+
+ opTimer.stop("Finished " + getsPerThread + " random iterations (scanned = " + scanned + ") in %DURATION%");
+
+ nm.delete();
+ }
+ };
+
+ Thread t = new Thread(r);
+ t.start();
+
+ threads.add(t);
+ }
+
+ for (Thread thread : threads) {
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private static void testLotsOfMapDeletes(final boolean doRemoves) {
+ final int numThreads = 8;
+ final int rowRange = 10000;
+ final int mapsPerThread = 50;
+ final int totalInserts = 100000000;
+ final int insertsPerMapPerThread = (int) (totalInserts / (double) numThreads / mapsPerThread);
+
+ System.out.println("insertsPerMapPerThread " + insertsPerMapPerThread);
+
+ ArrayList<Thread> threads = new ArrayList<Thread>();
+
+ for (int i = 0; i < numThreads; i++) {
+ Runnable r = new Runnable() {
+ @Override
+ public void run() {
+
+ int inserts = 0;
+ int removes = 0;
+
+ for (int i = 0; i < mapsPerThread; i++) {
+
+ NativeMap nm = new NativeMap();
+
+ for (int j = 0; j < insertsPerMapPerThread; j++) {
+ String row = String.format("r%08d", j % rowRange);
+ String val = row + "v";
+ put(nm, row, val, j);
+ inserts++;
+ }
+
+ if (doRemoves) {
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+ while (iter.hasNext()) {
+ iter.next();
+ iter.remove();
+ removes++;
+ }
+ }
+
+ nm.delete();
+ }
+
+ System.out.println("inserts " + inserts + " removes " + removes + " " + Thread.currentThread().getName());
+ }
+ };
+
+ Thread t = new Thread(r);
+ t.start();
+
+ threads.add(t);
+ }
+
+ for (Thread thread : threads) {
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private static void testLotsOfOverwrites() {
+ final Map<Integer,NativeMap> nativeMaps = new HashMap<Integer,NativeMap>();
+
+ int numThreads = 8;
+ final int insertsPerThread = (int) (100000000 / (double) numThreads);
+ final int rowRange = 10000;
+ final int numMaps = 50;
+
+ ArrayList<Thread> threads = new ArrayList<Thread>();
+
+ for (int i = 0; i < numThreads; i++) {
+ Runnable r = new Runnable() {
+ @Override
+ public void run() {
+ Random r = new Random();
+ int inserts = 0;
+
+ for (int i = 0; i < insertsPerThread / 100.0; i++) {
+ int map = r.nextInt(numMaps);
+
+ NativeMap nm;
+
+ synchronized (nativeMaps) {
+ nm = nativeMaps.get(map);
+ if (nm == null) {
+ nm = new NativeMap();
+ nativeMaps.put(map, nm);
+
+ }
+ }
+
+ synchronized (nm) {
+ for (int j = 0; j < 100; j++) {
+ String row = String.format("r%08d", r.nextInt(rowRange));
+ String val = row + "v";
+ put(nm, row, val, j);
+ inserts++;
+ }
+ }
+ }
+
+ System.out.println("inserts " + inserts + " " + Thread.currentThread().getName());
+ }
+ };
+
+ Thread t = new Thread(r);
+ t.start();
+
+ threads.add(t);
+ }
+
+ for (Thread thread : threads) {
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ }
+ }
+
+ Set<Entry<Integer,NativeMap>> es = nativeMaps.entrySet();
+ for (Entry<Integer,NativeMap> entry : es) {
+ entry.getValue().delete();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
index a35ca66,0000000..0d52f12
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
@@@ -1,181 -1,0 +1,180 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.continuous;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
+import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.server.util.reflection.CounterUtils;
+import org.apache.accumulo.test.continuous.ContinuousIngest.BaseOpts;
+import org.apache.accumulo.test.continuous.ContinuousIngest.ShortConverter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.validators.PositiveInteger;
+
+/**
+ * A map only job that reads a table created by continuous ingest and creates doubly linked list. This map reduce job tests the ability of a map only job to
+ * read and write to accumulo at the same time. This map reduce job mutates the table in such a way that it should not create any undefined nodes.
+ *
+ */
+public class ContinuousMoru extends Configured implements Tool {
+ private static final String PREFIX = ContinuousMoru.class.getSimpleName() + ".";
+ private static final String MAX_CQ = PREFIX + "MAX_CQ";
+ private static final String MAX_CF = PREFIX + "MAX_CF";
+ private static final String MAX = PREFIX + "MAX";
+ private static final String MIN = PREFIX + "MIN";
+ private static final String CI_ID = PREFIX + "CI_ID";
+
+ static enum Counts {
+ SELF_READ;
+ }
+
+ public static class CMapper extends Mapper<Key,Value,Text,Mutation> {
+
+ private short max_cf;
+ private short max_cq;
+ private Random random;
+ private String ingestInstanceId;
+ private byte[] iiId;
+ private long count;
+
+ private static final ColumnVisibility EMPTY_VIS = new ColumnVisibility();
+
+ @Override
+ public void setup(Context context) throws IOException, InterruptedException {
+ int max_cf = context.getConfiguration().getInt(MAX_CF, -1);
+ int max_cq = context.getConfiguration().getInt(MAX_CQ, -1);
+
+ if (max_cf > Short.MAX_VALUE || max_cq > Short.MAX_VALUE)
+ throw new IllegalArgumentException();
+
+ this.max_cf = (short) max_cf;
+ this.max_cq = (short) max_cq;
+
+ random = new Random();
+ ingestInstanceId = context.getConfiguration().get(CI_ID);
+ iiId = ingestInstanceId.getBytes(Constants.UTF8);
+
+ count = 0;
+ }
+
+ @Override
+ public void map(Key key, Value data, Context context) throws IOException, InterruptedException {
+
+ ContinuousWalk.validate(key, data);
+
+ if (WritableComparator.compareBytes(iiId, 0, iiId.length, data.get(), 0, iiId.length) != 0) {
+ // only rewrite data not written by this M/R job
+ byte[] val = data.get();
+
+ int offset = ContinuousWalk.getPrevRowOffset(val);
+ if (offset > 0) {
+ long rowLong = Long.parseLong(new String(val, offset, 16, Constants.UTF8), 16);
+ Mutation m = ContinuousIngest.genMutation(rowLong, random.nextInt(max_cf), random.nextInt(max_cq), EMPTY_VIS, iiId, count++, key.getRowData()
+ .toArray(), random, true);
+ context.write(null, m);
+ }
+
+ } else {
+ CounterUtils.increment(context.getCounter(Counts.SELF_READ));
+ }
+ }
+ }
+
+ static class Opts extends BaseOpts {
+ @Parameter(names = "--maxColF", description = "maximum column family value to use", converter=ShortConverter.class)
+ short maxColF = Short.MAX_VALUE;
+
+ @Parameter(names = "--maxColQ", description = "maximum column qualifier value to use", converter=ShortConverter.class)
+ short maxColQ = Short.MAX_VALUE;
+
+ @Parameter(names = "--maxMappers", description = "the maximum number of mappers to use", required = true, validateWith = PositiveInteger.class)
+ int maxMaps = 0;
+ }
+
+ @Override
+ public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException, AccumuloSecurityException {
+ Opts opts = new Opts();
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(ContinuousMoru.class.getName(), args, bwOpts);
+
+ Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+ job.setJarByClass(this.getClass());
+
+ job.setInputFormatClass(AccumuloInputFormat.class);
+ opts.setAccumuloConfigs(job);
+
+ // set up ranges
+ try {
+ Set<Range> ranges = opts.getConnector().tableOperations().splitRangeByTablets(opts.getTableName(), new Range(), opts.maxMaps);
+ AccumuloInputFormat.setRanges(job, ranges);
+ AccumuloInputFormat.setAutoAdjustRanges(job, false);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ job.setMapperClass(CMapper.class);
+
+ job.setNumReduceTasks(0);
+
+ job.setOutputFormatClass(AccumuloOutputFormat.class);
+ AccumuloOutputFormat.setBatchWriterOptions(job, bwOpts.getBatchWriterConfig());
+
+ Configuration conf = job.getConfiguration();
+ conf.setLong(MIN, opts.min);
+ conf.setLong(MAX, opts.max);
+ conf.setInt(MAX_CF, opts.maxColF);
+ conf.setInt(MAX_CQ, opts.maxColQ);
+ conf.set(CI_ID, UUID.randomUUID().toString());
+
+ job.waitForCompletion(true);
+ opts.stopTracing();
+ return job.isSuccessful() ? 0 : 1;
+ }
+
+ /**
+ *
+ * @param args
+ * instanceName zookeepers username password table columns outputpath
- * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(CachedConfiguration.getInstance(), new ContinuousMoru(), args);
+ if (res != 0)
+ System.exit(res);
+ }
+}
[18/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
index 06b9a7c,0000000..70156b2
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
@@@ -1,223 -1,0 +1,222 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.continuous;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.ClientOnDefaultTable;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.server.util.reflection.CounterUtils;
+import org.apache.accumulo.test.continuous.ContinuousWalk.BadChecksumException;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.VLongWritable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.validators.PositiveInteger;
+
+/**
+ * A map reduce job that verifies a table created by continuous ingest. It verifies that all referenced nodes are defined.
+ */
+
+public class ContinuousVerify extends Configured implements Tool {
+ public static final VLongWritable DEF = new VLongWritable(-1);
+
+ public static class CMapper extends Mapper<Key,Value,LongWritable,VLongWritable> {
+
+ private LongWritable row = new LongWritable();
+ private LongWritable ref = new LongWritable();
+ private VLongWritable vrow = new VLongWritable();
+
+ private long corrupt = 0;
+
+ @Override
+ public void map(Key key, Value data, Context context) throws IOException, InterruptedException {
+ long r = Long.parseLong(key.getRow().toString(), 16);
+ if (r < 0)
+ throw new IllegalArgumentException();
+
+ try {
+ ContinuousWalk.validate(key, data);
+ } catch (BadChecksumException bce) {
+ CounterUtils.increment(context.getCounter(Counts.CORRUPT));
+ if (corrupt < 1000) {
+ System.out.println("ERROR Bad checksum : " + key);
+ } else if (corrupt == 1000) {
+ System.out.println("Too many bad checksums, not printing anymore!");
+ }
+ corrupt++;
+ return;
+ }
+
+ row.set(r);
+
+ context.write(row, DEF);
+ byte[] val = data.get();
+
+ int offset = ContinuousWalk.getPrevRowOffset(val);
+ if (offset > 0) {
+ ref.set(Long.parseLong(new String(val, offset, 16, Constants.UTF8), 16));
+ vrow.set(r);
+ context.write(ref, vrow);
+ }
+ }
+ }
+
+ public static enum Counts {
+ UNREFERENCED, UNDEFINED, REFERENCED, CORRUPT
+ }
+
+ public static class CReducer extends Reducer<LongWritable,VLongWritable,Text,Text> {
+ private ArrayList<Long> refs = new ArrayList<Long>();
+
+ @Override
+ public void reduce(LongWritable key, Iterable<VLongWritable> values, Context context) throws IOException, InterruptedException {
+
+ int defCount = 0;
+
+ refs.clear();
+ for (VLongWritable type : values) {
+ if (type.get() == -1) {
+ defCount++;
+ } else {
+ refs.add(type.get());
+ }
+ }
+
+ if (defCount == 0 && refs.size() > 0) {
+ StringBuilder sb = new StringBuilder();
+ String comma = "";
+ for (Long ref : refs) {
+ sb.append(comma);
+ comma = ",";
+ sb.append(new String(ContinuousIngest.genRow(ref), Constants.UTF8));
+ }
+
+ context.write(new Text(ContinuousIngest.genRow(key.get())), new Text(sb.toString()));
+ CounterUtils.increment(context.getCounter(Counts.UNDEFINED));
+
+ } else if (defCount > 0 && refs.size() == 0) {
+ CounterUtils.increment(context.getCounter(Counts.UNREFERENCED));
+ } else {
+ CounterUtils.increment(context.getCounter(Counts.REFERENCED));
+ }
+
+ }
+ }
+
+ static class Opts extends ClientOnDefaultTable {
+ @Parameter(names = "--output", description = "location in HDFS to store the results; must not exist", required = true)
+ String outputDir = "/tmp/continuousVerify";
+
+ @Parameter(names = "--maxMappers", description = "the maximum number of mappers to use", required = true, validateWith = PositiveInteger.class)
+ int maxMaps = 0;
+
+ @Parameter(names = "--reducers", description = "the number of reducers to use", required = true, validateWith = PositiveInteger.class)
+ int reducers = 0;
+
+ @Parameter(names = "--offline", description = "perform the verification directly on the files while the table is offline")
+ boolean scanOffline = false;
+
+ public Opts() {
+ super("ci");
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(this.getClass().getName(), args);
+
+ Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+ job.setJarByClass(this.getClass());
+
+ job.setInputFormatClass(AccumuloInputFormat.class);
+ opts.setAccumuloConfigs(job);
+
+ Set<Range> ranges = null;
+ String clone = opts.getTableName();
+ Connector conn = null;
+
+ if (opts.scanOffline) {
+ Random random = new Random();
+ clone = opts.getTableName() + "_" + String.format("%016x", (random.nextLong() & 0x7fffffffffffffffl));
+ conn = opts.getConnector();
+ conn.tableOperations().clone(opts.getTableName(), clone, true, new HashMap<String,String>(), new HashSet<String>());
+ ranges = conn.tableOperations().splitRangeByTablets(opts.getTableName(), new Range(), opts.maxMaps);
+ conn.tableOperations().offline(clone);
+ AccumuloInputFormat.setInputTableName(job, clone);
+ AccumuloInputFormat.setOfflineTableScan(job, true);
+ } else {
+ ranges = opts.getConnector().tableOperations().splitRangeByTablets(opts.getTableName(), new Range(), opts.maxMaps);
+ }
+
+ AccumuloInputFormat.setRanges(job, ranges);
+ AccumuloInputFormat.setAutoAdjustRanges(job, false);
+
+ job.setMapperClass(CMapper.class);
+ job.setMapOutputKeyClass(LongWritable.class);
+ job.setMapOutputValueClass(VLongWritable.class);
+
+ job.setReducerClass(CReducer.class);
+ job.setNumReduceTasks(opts.reducers);
+
+ job.setOutputFormatClass(TextOutputFormat.class);
+
+ job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", opts.scanOffline);
+
+ TextOutputFormat.setOutputPath(job, new Path(opts.outputDir));
+
+ job.waitForCompletion(true);
+
+ if (opts.scanOffline) {
+ conn.tableOperations().delete(clone);
+ }
+ opts.stopTracing();
+ return job.isSuccessful() ? 0 : 1;
+ }
+
+ /**
+ *
+ * @param args
+ * instanceName zookeepers username password table columns outputpath
- * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(CachedConfiguration.getInstance(), new ContinuousVerify(), args);
+ if (res != 0)
+ System.exit(res);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/functional/CacheTestClean.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/CacheTestClean.java
index c522914,0000000..f1dfcd2
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/CacheTestClean.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/CacheTestClean.java
@@@ -1,51 -1,0 +1,48 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import java.io.File;
+import java.util.Arrays;
+
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+
+public class CacheTestClean {
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ String rootDir = args[0];
+ File reportDir = new File(args[1]);
+
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ if (zoo.exists(rootDir)) {
+ zoo.recursiveDelete(rootDir, NodeMissingPolicy.FAIL);
+ }
+
+ if (!reportDir.exists()) {
+ reportDir.mkdir();
+ } else {
+ File[] files = reportDir.listFiles();
+ if (files.length != 0)
+ throw new Exception("dir " + reportDir + " is not empty: " + Arrays.asList(files));
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
index 2b775c5,0000000..06c6fdb
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
@@@ -1,217 -1,0 +1,213 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.server.util.reflection.CounterUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Runs the functional tests via map-reduce.
+ *
+ * First, be sure everything is compiled.
+ *
+ * Second, get a list of the tests you want to run:
+ *
+ * <pre>
+ * $ python test/system/auto/run.py -l > tests
+ * </pre>
+ *
+ * Put the list of tests into HDFS:
+ *
+ * <pre>
+ * $ hadoop fs -put tests /user/hadoop/tests
+ * </pre>
+ *
+ * Run the map-reduce job:
+ *
+ * <pre>
+ * $ ./bin/accumulo accumulo.test.functional.RunTests --tests /user/hadoop/tests --output /user/hadoop/results
+ * </pre>
+ *
+ * Note that you will need to have some configuration in conf/accumulo-site.xml (to locate zookeeper). The map-reduce jobs will not use your local accumulo
+ * instance.
+ *
+ */
+public class RunTests extends Configured implements Tool {
+
+ static final public String JOB_NAME = "Functional Test Runner";
+ private static final Logger log = Logger.getLogger(RunTests.class);
+
+ private Job job = null;
+
+ private static final int DEFAULT_TIMEOUT_FACTOR = 1;
+
+ static class Opts extends Help {
+ @Parameter(names="--tests", description="newline separated list of tests to run", required=true)
+ String testFile;
+ @Parameter(names="--output", description="destination for the results of tests in HDFS", required=true)
+ String outputPath;
+ @Parameter(names="--timeoutFactor", description="Optional scaling factor for timeout for both mapred.task.timeout and -f flag on run.py", required=false)
+ Integer intTimeoutFactor = DEFAULT_TIMEOUT_FACTOR;
+ }
+
+ static final String TIMEOUT_FACTOR = RunTests.class.getName() + ".timeoutFactor";
+
+ static public class TestMapper extends Mapper<LongWritable,Text,Text,Text> {
+
+ private static final String REDUCER_RESULT_START = "::::: ";
+ private static final int RRS_LEN = REDUCER_RESULT_START.length();
+ private Text result = new Text();
+ String mapperTimeoutFactor = null;
+
+ private static enum Outcome {
+ SUCCESS, FAILURE, ERROR, UNEXPECTED_SUCCESS, EXPECTED_FAILURE
+ }
+ private static final Map<Character, Outcome> OUTCOME_COUNTERS;
+ static {
+ OUTCOME_COUNTERS = new java.util.HashMap<Character, Outcome>();
+ OUTCOME_COUNTERS.put('S', Outcome.SUCCESS);
+ OUTCOME_COUNTERS.put('F', Outcome.FAILURE);
+ OUTCOME_COUNTERS.put('E', Outcome.ERROR);
+ OUTCOME_COUNTERS.put('T', Outcome.UNEXPECTED_SUCCESS);
+ OUTCOME_COUNTERS.put('G', Outcome.EXPECTED_FAILURE);
+ }
+
+ @Override
+ protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
+ List<String> cmd = Arrays.asList("/usr/bin/python", "test/system/auto/run.py", "-m", "-f", mapperTimeoutFactor, "-t", value.toString());
+ log.info("Running test " + cmd);
+ ProcessBuilder pb = new ProcessBuilder(cmd);
+ pb.directory(new File(context.getConfiguration().get("accumulo.home")));
+ pb.redirectErrorStream(true);
+ Process p = pb.start();
+ p.getOutputStream().close();
+ InputStream out = p.getInputStream();
+ InputStreamReader outr = new InputStreamReader(out, Constants.UTF8);
+ BufferedReader br = new BufferedReader(outr);
+ String line;
+ try {
+ while ((line = br.readLine()) != null) {
+ log.info("More: " + line);
+ if (line.startsWith(REDUCER_RESULT_START)) {
+ String resultLine = line.substring(RRS_LEN);
+ if (resultLine.length() > 0) {
+ Outcome outcome = OUTCOME_COUNTERS.get(resultLine.charAt(0));
+ if (outcome != null) {
+ CounterUtils.increment(context.getCounter(outcome));
+ }
+ }
+ String taskAttemptId = context.getTaskAttemptID().toString();
+ result.set(taskAttemptId + " " + resultLine);
+ context.write(value, result);
+ }
+ }
+ } catch (Exception ex) {
+ log.error(ex);
+ context.progress();
+ }
+
+ p.waitFor();
+ }
+
+ @Override
+ protected void setup(Mapper<LongWritable,Text,Text,Text>.Context context) throws IOException, InterruptedException {
+ mapperTimeoutFactor = Integer.toString(context.getConfiguration().getInt(TIMEOUT_FACTOR, DEFAULT_TIMEOUT_FACTOR));
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ job = new Job(getConf(), JOB_NAME);
+ job.setJarByClass(this.getClass());
+ Opts opts = new Opts();
+ opts.parseArgs(RunTests.class.getName(), args);
+
+ // this is like 1-2 tests per mapper
+ Configuration conf = job.getConfiguration();
+ conf.setInt("mapred.max.split.size", 40);
+ conf.set("accumulo.home", System.getenv("ACCUMULO_HOME"));
+
+ // Taking third argument as scaling factor to setting mapred.task.timeout
+ // and TIMEOUT_FACTOR
+ conf.setInt("mapred.task.timeout", opts.intTimeoutFactor * 8 * 60 * 1000);
+ conf.setInt(TIMEOUT_FACTOR, opts.intTimeoutFactor);
+ conf.setBoolean("mapred.map.tasks.speculative.execution", false);
+
+ // set input
+ job.setInputFormatClass(TextInputFormat.class);
+ TextInputFormat.setInputPaths(job, new Path(opts.testFile));
+
+ // set output
+ job.setOutputFormatClass(TextOutputFormat.class);
+ FileSystem fs = FileSystem.get(conf);
+ Path destination = new Path(opts.outputPath);
+ if (fs.exists(destination)) {
+ log.info("Deleting existing output directory " + opts.outputPath);
+ fs.delete(destination, true);
+ }
+ TextOutputFormat.setOutputPath(job, destination);
+
+ // configure default reducer: put the results into one file
+ job.setNumReduceTasks(1);
+
+ // set mapper
+ job.setMapperClass(TestMapper.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(Text.class);
+
+ // don't do anything with the results (yet) a summary would be nice
+ job.setNumReduceTasks(0);
+
+ // submit the job
+ log.info("Starting tests");
+ return 0;
+ }
+
- /**
- * @param args
- * @throws Exception
- */
+ public static void main(String[] args) throws Exception {
+ RunTests tests = new RunTests();
+ ToolRunner.run(new Configuration(), tests, args);
+ tests.job.waitForCompletion(true);
+ if (!tests.job.isSuccessful())
+ System.exit(1);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
index a9b072e,0000000..85cddbb
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
@@@ -1,246 -1,0 +1,243 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.performance.metadata;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.TreeSet;
+import java.util.UUID;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.util.Stat;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.hadoop.io.Text;
+
+/**
+ * This little program can be used to write a lot of entries to the !METADATA table and measure the performance of varying numbers of threads doing !METADATA
+ * lookups using the batch scanner.
+ *
+ *
+ */
+
+public class MetadataBatchScanTest {
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+
+ final Connector connector = new ZooKeeperInstance("acu14", "localhost")
+ .getConnector(SecurityConstants.SYSTEM_PRINCIPAL, SecurityConstants.getSystemToken());
+
+ TreeSet<Long> splits = new TreeSet<Long>();
+ Random r = new Random(42);
+
+ while (splits.size() < 99999) {
+ splits.add((r.nextLong() & 0x7fffffffffffffffl) % 1000000000000l);
+ }
+
+ Text tid = new Text("8");
+ Text per = null;
+
+ ArrayList<KeyExtent> extents = new ArrayList<KeyExtent>();
+
+ for (Long split : splits) {
+ Text er = new Text(String.format("%012d", split));
+ KeyExtent ke = new KeyExtent(tid, er, per);
+ per = er;
+
+ extents.add(ke);
+ }
+
+ extents.add(new KeyExtent(tid, null, per));
+
+ if (args[0].equals("write")) {
+
+ BatchWriter bw = connector.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig());
+
+ for (KeyExtent extent : extents) {
+ Mutation mut = extent.getPrevRowUpdateMutation();
+ new TServerInstance(AddressUtil.parseAddress("192.168.1.100", 4567), "DEADBEEF").putLocation(mut);
+ bw.addMutation(mut);
+ }
+
+ bw.close();
+ } else if (args[0].equals("writeFiles")) {
+ BatchWriter bw = connector.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig());
+
+ for (KeyExtent extent : extents) {
+
+ Mutation mut = new Mutation(extent.getMetadataEntry());
+
+ String dir = "/t-" + UUID.randomUUID();
+
+ Constants.METADATA_DIRECTORY_COLUMN.put(mut, new Value(dir.getBytes(Constants.UTF8)));
+
+ for (int i = 0; i < 5; i++) {
+ mut.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(dir + "/00000_0000" + i + ".map"), new Value("10000,1000000".getBytes(Constants.UTF8)));
+ }
+
+ bw.addMutation(mut);
+ }
+
+ bw.close();
+ } else if (args[0].equals("scan")) {
+
+ int numThreads = Integer.parseInt(args[1]);
+ final int numLoop = Integer.parseInt(args[2]);
+ int numLookups = Integer.parseInt(args[3]);
+
+ HashSet<Integer> indexes = new HashSet<Integer>();
+ while (indexes.size() < numLookups) {
+ indexes.add(r.nextInt(extents.size()));
+ }
+
+ final List<Range> ranges = new ArrayList<Range>();
+ for (Integer i : indexes) {
+ ranges.add(extents.get(i).toMetadataRange());
+ }
+
+ Thread threads[] = new Thread[numThreads];
+
+ for (int i = 0; i < threads.length; i++) {
+ threads[i] = new Thread(new Runnable() {
+
+ @Override
+ public void run() {
+ try {
+ System.out.println(runScanTest(connector, numLoop, ranges));
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ });
+ }
+
+ long t1 = System.currentTimeMillis();
+
+ for (int i = 0; i < threads.length; i++) {
+ threads[i].start();
+ }
+
+ for (int i = 0; i < threads.length; i++) {
+ threads[i].join();
+ }
+
+ long t2 = System.currentTimeMillis();
+
+ System.out.printf("tt : %6.2f%n", (t2 - t1) / 1000.0);
+
+ } else {
+ throw new IllegalArgumentException();
+ }
+
+ }
+
+ private static ScanStats runScanTest(Connector connector, int numLoop, List<Range> ranges) throws Exception {
+ Scanner scanner = null;/*
+ * connector.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS); ColumnFQ.fetch(scanner,
+ * Constants.METADATA_LOCATION_COLUMN); ColumnFQ.fetch(scanner, Constants.METADATA_PREV_ROW_COLUMN);
+ */
+
+ BatchScanner bs = connector.createBatchScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS, 1);
+ bs.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
+ Constants.METADATA_PREV_ROW_COLUMN.fetch(bs);
+
+ bs.setRanges(ranges);
+
+ // System.out.println(ranges);
+
+ ScanStats stats = new ScanStats();
+ for (int i = 0; i < numLoop; i++) {
+ ScanStat ss = scan(bs, ranges, scanner);
+ stats.merge(ss);
+ }
+
+ return stats;
+ }
+
+ private static class ScanStat {
+ long delta1;
+ long delta2;
+ int count1;
+ int count2;
+ }
+
+ private static class ScanStats {
+ Stat delta1 = new Stat();
+ Stat delta2 = new Stat();
+ Stat count1 = new Stat();
+ Stat count2 = new Stat();
+
+ void merge(ScanStat ss) {
+ delta1.addStat(ss.delta1);
+ delta2.addStat(ss.delta2);
+ count1.addStat(ss.count1);
+ count2.addStat(ss.count2);
+ }
+
+ @Override
+ public String toString() {
+ return "[" + delta1 + "] [" + delta2 + "]";
+ }
+ }
+
+ private static ScanStat scan(BatchScanner bs, List<Range> ranges, Scanner scanner) {
+
+ // System.out.println("ranges : "+ranges);
+
+ ScanStat ss = new ScanStat();
+
+ long t1 = System.currentTimeMillis();
+ int count = 0;
+ for (@SuppressWarnings("unused")
+ Entry<Key,Value> entry : bs) {
+ count++;
+ }
+ long t2 = System.currentTimeMillis();
+
+ ss.delta1 = (t2 - t1);
+ ss.count1 = count;
+
+ count = 0;
+ t1 = System.currentTimeMillis();
+ /*
+ * for (Range range : ranges) { scanner.setRange(range); for (Entry<Key, Value> entry : scanner) { count++; } }
+ */
+
+ t2 = System.currentTimeMillis();
+
+ ss.delta2 = (t2 - t1);
+ ss.count2 = count;
+
+ return ss;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/performance/thrift/NullTserver.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/performance/thrift/NullTserver.java
index 71970d3,0000000..e6fcd5b
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/performance/thrift/NullTserver.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/thrift/NullTserver.java
@@@ -1,258 -1,0 +1,252 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.performance.thrift;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
- import org.apache.accumulo.trace.thrift.TInfo;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.thrift.InitialMultiScan;
+import org.apache.accumulo.core.data.thrift.InitialScan;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.data.thrift.MapFileInfo;
+import org.apache.accumulo.core.data.thrift.MultiScanResult;
+import org.apache.accumulo.core.data.thrift.ScanResult;
+import org.apache.accumulo.core.data.thrift.TColumn;
+import org.apache.accumulo.core.data.thrift.TConstraintViolationSummary;
+import org.apache.accumulo.core.data.thrift.TKeyExtent;
+import org.apache.accumulo.core.data.thrift.TMutation;
+import org.apache.accumulo.core.data.thrift.TRange;
+import org.apache.accumulo.core.data.thrift.UpdateErrors;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Processor;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.client.ClientServiceHandler;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.master.state.Assignment;
+import org.apache.accumulo.server.master.state.MetaDataStateStore;
+import org.apache.accumulo.server.master.state.MetaDataTableScanner;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletLocationState;
+import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.accumulo.server.util.TServerUtils;
+import org.apache.accumulo.server.zookeeper.TransactionWatcher;
++import org.apache.accumulo.trace.thrift.TInfo;
+import org.apache.hadoop.io.Text;
+import org.apache.thrift.TException;
+
+import com.beust.jcommander.Parameter;
+
+
+/**
+ * The purpose of this class is to server as fake tserver that is a data sink like /dev/null. NullTserver modifies the !METADATA location entries for a table to
+ * point to it. This allows thrift performance to be measured by running any client code that writes to a table.
+ *
+ */
+
+public class NullTserver {
+
+ public static class ThriftClientHandler extends ClientServiceHandler implements TabletClientService.Iface {
+
+ private long updateSession = 1;
+
+ public ThriftClientHandler(Instance instance, TransactionWatcher watcher) {
+ super(instance, watcher);
+ }
+
+ @Override
+ public long startUpdate(TInfo tinfo, TCredentials credentials) {
+ return updateSession++;
+ }
+
+ @Override
+ public void applyUpdates(TInfo tinfo, long updateID, TKeyExtent keyExtent, List<TMutation> mutation) {}
+
+ @Override
+ public UpdateErrors closeUpdate(TInfo tinfo, long updateID) {
+ return new UpdateErrors(new HashMap<TKeyExtent,Long>(), new ArrayList<TConstraintViolationSummary>(), new HashMap<TKeyExtent, SecurityErrorCode>());
+ }
+
+ @Override
+ public List<TKeyExtent> bulkImport(TInfo tinfo, TCredentials credentials, long tid, Map<TKeyExtent,Map<String,MapFileInfo>> files, boolean setTime) {
+ return null;
+ }
+
+ @Override
+ public void closeMultiScan(TInfo tinfo, long scanID) {}
+
+ @Override
+ public void closeScan(TInfo tinfo, long scanID) {}
+
+ @Override
+ public MultiScanResult continueMultiScan(TInfo tinfo, long scanID) {
+ return null;
+ }
+
+ @Override
+ public ScanResult continueScan(TInfo tinfo, long scanID) {
+ return null;
+ }
+
+ @Override
+ public void splitTablet(TInfo tinfo, TCredentials credentials, TKeyExtent extent, ByteBuffer splitPoint) {
+
+ }
+
+ @Override
+ public InitialMultiScan startMultiScan(TInfo tinfo, TCredentials credentials, Map<TKeyExtent,List<TRange>> batch, List<TColumn> columns,
+ List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites) {
+ return null;
+ }
+
+ @Override
+ public InitialScan startScan(TInfo tinfo, TCredentials credentials, TKeyExtent extent, TRange range, List<TColumn> columns, int batchSize,
+ List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites, boolean isolated) {
+ return null;
+ }
+
+ @Override
+ public void update(TInfo tinfo, TCredentials credentials, TKeyExtent keyExtent, TMutation mutation) {
+
+ }
+
+ @Override
+ public TabletServerStatus getTabletServerStatus(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ return null;
+ }
+
+ @Override
+ public List<TabletStats> getTabletStats(TInfo tinfo, TCredentials credentials, String tableId) throws ThriftSecurityException, TException {
+ return null;
+ }
+
+ @Override
+ public TabletStats getHistoricalStats(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ return null;
+ }
+
+ @Override
+ public void halt(TInfo tinfo, TCredentials credentials, String lock) throws ThriftSecurityException, TException {}
+
+ @Override
+ public void fastHalt(TInfo tinfo, TCredentials credentials, String lock) {}
+
+ @Override
+ public void loadTablet(TInfo tinfo, TCredentials credentials, String lock, TKeyExtent extent) throws TException {}
+
+ @Override
+ public void unloadTablet(TInfo tinfo, TCredentials credentials, String lock, TKeyExtent extent, boolean save) throws TException {}
+
+ @Override
+ public List<ActiveScan> getActiveScans(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ return new ArrayList<ActiveScan>();
+ }
+
+ @Override
+ public void chop(TInfo tinfo, TCredentials credentials, String lock, TKeyExtent extent) throws TException {}
+
+ @Override
+ public void flushTablet(TInfo tinfo, TCredentials credentials, String lock, TKeyExtent extent) throws TException {
+
+ }
+
+ @Override
+ public void compact(TInfo tinfo, TCredentials credentials, String lock, String tableId, ByteBuffer startRow, ByteBuffer endRow) throws TException {
+
+ }
+
+ @Override
+ public void flush(TInfo tinfo, TCredentials credentials, String lock, String tableId, ByteBuffer startRow, ByteBuffer endRow) throws TException {
+
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface#removeLogs(org.apache.accumulo.trace.thrift.TInfo,
- * org.apache.accumulo.core.security.thrift.Credentials, java.util.List)
- */
+ @Override
+ public void removeLogs(TInfo tinfo, TCredentials credentials, List<String> filenames) throws TException {
+ }
+
+ @Override
+ public List<ActiveCompaction> getActiveCompactions(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ return new ArrayList<ActiveCompaction>();
+ }
+ }
+
+ static class Opts extends Help {
+ @Parameter(names={"-i", "--instance"}, description="instance name", required=true)
+ String iname = null;
+ @Parameter(names={"-z", "--keepers"}, description="comma-separated list of zookeeper host:ports", required=true)
+ String keepers = null;
+ @Parameter(names="--table", description="table to adopt", required=true)
+ String tableName = null;
+ @Parameter(names="--port", description="port number to use")
+ int port = DefaultConfiguration.getInstance().getPort(Property.TSERV_CLIENTPORT);
+ }
+
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(NullTserver.class.getName(), args);
+
+ TransactionWatcher watcher = new TransactionWatcher();
+ ThriftClientHandler tch = new ThriftClientHandler(HdfsZooInstance.getInstance(), watcher);
+ Processor<Iface> processor = new Processor<Iface>(tch);
+ TServerUtils.startTServer(opts.port, processor, "NullTServer", "null tserver", 2, 1000, 10*1024*1024);
+
+ InetSocketAddress addr = new InetSocketAddress(InetAddress.getLocalHost(), opts.port);
+
+ // modify !METADATA
+ ZooKeeperInstance zki = new ZooKeeperInstance(opts.iname, opts.keepers);
+ String tableId = Tables.getTableId(zki, opts.tableName);
+
+ // read the locations for the table
+ Range tableRange = new KeyExtent(new Text(tableId), null, null).toMetadataRange();
+ MetaDataTableScanner s = new MetaDataTableScanner(zki, SecurityConstants.getSystemCredentials(), tableRange);
+ long randomSessionID = opts.port;
+ TServerInstance instance = new TServerInstance(addr, randomSessionID);
+ List<Assignment> assignments = new ArrayList<Assignment>();
+ while (s.hasNext()) {
+ TabletLocationState next = s.next();
+ assignments.add(new Assignment(next.extent, instance));
+ }
+ s.close();
+ // point them to this server
+ MetaDataStateStore store = new MetaDataStateStore();
+ store.setLocations(assignments);
+
+ while (true) {
+ UtilWaitThread.sleep(10000);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/randomwalk/Framework.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/Framework.java
index 9d01929,0000000..7cb58c9
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Framework.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Framework.java
@@@ -1,129 -1,0 +1,126 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.randomwalk;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.util.HashMap;
+import java.util.Properties;
+
+import org.apache.log4j.Logger;
+import org.apache.log4j.xml.DOMConfigurator;
+
+import com.beust.jcommander.Parameter;
+
+public class Framework {
+
+ private static final Logger log = Logger.getLogger(Framework.class);
+ private HashMap<String,Node> nodes = new HashMap<String,Node>();
+ private String configDir = null;
+ private static final Framework INSTANCE = new Framework();
+
+ /**
+ * @return Singleton instance of Framework
+ */
+ public static Framework getInstance() {
+ return INSTANCE;
+ }
+
+ public String getConfigDir() {
+ return configDir;
+ }
+
+ public void setConfigDir(String confDir) {
+ configDir = confDir;
+ }
+
+ /**
+ * Run random walk framework
+ *
+ * @param startName
+ * Full name of starting graph or test
- * @param state
- * @param confDir
+ */
+ public int run(String startName, State state, String confDir) {
+
+ try {
+ System.out.println("confDir " + confDir);
+ setConfigDir(confDir);
+ Node node = getNode(startName);
+ node.visit(state, new Properties());
+ } catch (Exception e) {
+ log.error("Error during random walk", e);
+ return -1;
+ }
+ return 0;
+ }
+
+ /**
+ * Creates node (if it does not already exist) and inserts into map
+ *
+ * @param id
+ * Name of node
+ * @return Node specified by id
- * @throws Exception
+ */
+ public Node getNode(String id) throws Exception {
+
+ // check for node in nodes
+ if (nodes.containsKey(id)) {
+ return nodes.get(id);
+ }
+
+ // otherwise create and put in nodes
+ Node node = null;
+ if (id.endsWith(".xml")) {
+ node = new Module(new File(configDir + "modules/" + id));
+ } else {
+ node = (Test) Class.forName(id).newInstance();
+ }
+ nodes.put(id, node);
+ return node;
+ }
+
+ static class Opts extends org.apache.accumulo.core.cli.Help {
+ @Parameter(names="--configDir", required=true, description="directory containing the test configuration")
+ String configDir;
+ @Parameter(names="--logDir", required=true, description="location of the local logging directory")
+ String localLogPath;
+ @Parameter(names="--logId", required=true, description="a unique log identifier (like a hostname, or pid)")
+ String logId;
+ @Parameter(names="--module", required=true, description="the name of the module to run")
+ String module;
+ }
+
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(Framework.class.getName(), args);
+
+ Properties props = new Properties();
+ FileInputStream fis = new FileInputStream(opts.configDir + "/randomwalk.conf");
+ props.load(fis);
+ fis.close();
+
+ System.setProperty("localLog", opts.localLogPath + "/" + opts.logId);
+ System.setProperty("nfsLog", props.getProperty("NFS_LOGPATH") + "/" + opts.logId);
+
+ DOMConfigurator.configure(opts.configDir + "logger.xml");
+
+ State state = new State(props);
+ int retval = getInstance().run(opts.module, state, opts.configDir);
+
+ System.exit(retval);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
index 1868ade,0000000..b74b6cd
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
@@@ -1,64 -1,0 +1,63 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.randomwalk;
+
+import java.util.Properties;
+
+import org.apache.log4j.Logger;
+
+/**
+ * Represents a point in graph of RandomFramework
+ */
+public abstract class Node {
+
+ protected final Logger log = Logger.getLogger(this.getClass());
+ long progress = System.currentTimeMillis();
+
+ /**
+ * Visits node
+ *
+ * @param state
+ * Random walk state passed between nodes
- * @throws Exception
+ */
+ public abstract void visit(State state, Properties props) throws Exception;
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null)
+ return false;
+ return toString().equals(o.toString());
+ }
+
+ @Override
+ public String toString() {
+ return this.getClass().getName();
+ }
+
+ @Override
+ public int hashCode() {
+ return toString().hashCode();
+ }
+
+ synchronized public void makingProgress() {
+ progress = System.currentTimeMillis();
+ }
+
+ synchronized public long lastProgress() {
+ return progress;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
index a0dd37c,0000000..4581b04
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
@@@ -1,110 -1,0 +1,107 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.randomwalk.concurrent;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.test.randomwalk.State;
+import org.apache.accumulo.test.randomwalk.Test;
+
+/**
+ *
+ */
+public class CheckBalance extends Test {
+
+ static final String LAST_UNBALANCED_TIME = "lastUnbalancedTime";
+ static final String UNBALANCED_COUNT = "unbalancedCount";
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.test.randomwalk.Node#visit(org.apache.accumulo.test.randomwalk.State, java.util.Properties)
- */
+ @Override
+ public void visit(State state, Properties props) throws Exception {
+ log.debug("checking balance");
+ Map<String,Long> counts = new HashMap<String,Long>();
+ Scanner scanner = state.getConnector().createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ scanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
+ for (Entry<Key,Value> entry : scanner) {
+ String location = entry.getKey().getColumnQualifier().toString();
+ Long count = counts.get(location);
+ if (count == null)
+ count = Long.valueOf(0);
+ counts.put(location, count + 1);
+ }
+ double total = 0.;
+ for (Long count : counts.values()) {
+ total += count.longValue();
+ }
+ final double average = total / counts.size();
+ final double sd = stddev(counts.values(), average);
+ log.debug("average " + average + ", standard deviation " + sd);
+
+ // Check for balanced # of tablets on each node
+ double maxDifference = 2.0 * sd;
+ String unbalancedLocation = null;
+ long lastCount = 0L;
+ boolean balanced = true;
+ for (Entry<String,Long> entry : counts.entrySet()) {
+ long thisCount = entry.getValue().longValue();
+ if (Math.abs(thisCount - average) > maxDifference) {
+ balanced = false;
+ log.debug("unbalanced: " + entry.getKey() + " has " + entry.getValue() + " tablets and the average is " + average);
+ unbalancedLocation = entry.getKey();
+ lastCount = thisCount;
+ }
+ }
+
+ // It is expected that the number of tablets will be uneven for short
+ // periods of time. Don't complain unless we've seen it only unbalanced
+ // over a 15 minute period and it's been at least three checks.
+ if (!balanced) {
+ Long last = state.getLong(LAST_UNBALANCED_TIME);
+ if (last != null && System.currentTimeMillis() - last > 15 * 60 * 1000) {
+ Integer count = state.getInteger(UNBALANCED_COUNT);
+ if (count == null)
+ count = Integer.valueOf(0);
+ if (count > 3)
+ throw new Exception("servers are unbalanced! location " + unbalancedLocation + " count " + lastCount + " too far from average " + average);
+ count++;
+ state.set(UNBALANCED_COUNT, count);
+ }
+ if (last == null)
+ state.set(LAST_UNBALANCED_TIME, System.currentTimeMillis());
+ } else {
+ state.remove(LAST_UNBALANCED_TIME);
+ state.remove(UNBALANCED_COUNT);
+ }
+ }
+
+ private static double stddev(Collection<Long> samples, double avg) {
+ int num = samples.size();
+ double sqrtotal = 0.0;
+ for (Long s : samples) {
+ double diff = s.doubleValue() - avg;
+ sqrtotal += diff * diff;
+ }
- return Math.sqrt(sqrtotal / (double) num);
++ return Math.sqrt(sqrtotal / num);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
----------------------------------------------------------------------
diff --cc trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
index 049b2a2,0000000..dfa9f0c
mode 100644,000000..100644
--- a/trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
+++ b/trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
@@@ -1,132 -1,0 +1,122 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.trace.instrument.receivers;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.ZooKeeper.States;
+
+/**
+ * Find a Span collector via zookeeper and push spans there via Thrift RPC
+ *
+ */
+public class ZooSpanClient extends SendSpansViaThrift {
+
+ private static final Logger log = Logger.getLogger(ZooSpanClient.class);
+ private static final int TOTAL_TIME_WAIT_CONNECT_MS = 10 * 1000;
+ private static final int TIME_WAIT_CONNECT_CHECK_MS = 100;
+ private static final Charset UTF8 = Charset.forName("UTF-8");
+
+ ZooKeeper zoo = null;
+ final String path;
+ final Random random = new Random();
+ final List<String> hosts = new ArrayList<String>();
+
+ public ZooSpanClient(String keepers, final String path, String host, String service, long millis) throws IOException, KeeperException, InterruptedException {
+ super(host, service, millis);
+ this.path = path;
+ zoo = new ZooKeeper(keepers, 30 * 1000, new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ try {
+ if (zoo != null) {
+ updateHosts(path, zoo.getChildren(path, null));
+ }
+ } catch (Exception ex) {
+ log.error("unable to get destination hosts in zookeeper", ex);
+ }
+ }
+ });
+ for (int i = 0; i < TOTAL_TIME_WAIT_CONNECT_MS; i += TIME_WAIT_CONNECT_CHECK_MS) {
+ if (zoo.getState().equals(States.CONNECTED))
+ break;
+ try {
+ Thread.sleep(TIME_WAIT_CONNECT_CHECK_MS);
+ } catch (InterruptedException ex) {
+ break;
+ }
+ }
+ zoo.getChildren(path, true);
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see trace.instrument.receivers.AsyncSpanReceiver#flush()
- */
+ @Override
+ public void flush() {
+ if (!hosts.isEmpty())
+ super.flush();
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see trace.instrument.receivers.AsyncSpanReceiver#sendSpans()
- */
+ @Override
+ void sendSpans() {
+ if (hosts.isEmpty()) {
+ if (!sendQueue.isEmpty()) {
+ log.error("No hosts to send data to, dropping queued spans");
+ synchronized (sendQueue) {
+ sendQueue.clear();
+ sendQueue.notifyAll();
+ }
+ }
+ } else {
+ super.sendSpans();
+ }
+ }
+
+ synchronized private void updateHosts(String path, List<String> children) {
+ log.debug("Scanning trace hosts in zookeeper: " + path);
+ try {
+ List<String> hosts = new ArrayList<String>();
+ for (String child : children) {
+ byte[] data = zoo.getData(path + "/" + child, null, null);
+ hosts.add(new String(data, UTF8));
+ }
+ this.hosts.clear();
+ this.hosts.addAll(hosts);
+ log.debug("Trace hosts: " + this.hosts);
+ } catch (Exception ex) {
+ log.error("unable to get destination hosts in zookeeper", ex);
+ }
+ }
+
+ @Override
+ synchronized protected String getSpanKey(Map<String,String> data) {
+ if (hosts.size() > 0) {
+ String host = hosts.get(random.nextInt(hosts.size()));
+ log.debug("sending data to " + host);
+ return host;
+ }
+ return null;
+ }
+}
[56/64] [abbrv] Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
index 5c7b857,0000000..53abbbe
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
@@@ -1,690 -1,0 +1,689 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.ClientSideIteratorScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableDeletedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.client.impl.OfflineScanner;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.TabletLocator;
+import org.apache.accumulo.core.client.mapreduce.lib.util.InputConfigurator;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * An abstract input format to provide shared methods common to all other input format classes. At the very least, any classes inheriting from this class will
+ * need to define their own {@link RecordReader}.
+ */
+public abstract class AbstractInputFormat<K,V> extends InputFormat<K,V> {
+
+ protected static final Class<?> CLASS = AccumuloInputFormat.class;
+ protected static final Logger log = Logger.getLogger(CLASS);
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
+ * conversion to a string, and is not intended to be secure.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission)
+ * @param token
+ * the user's password
- * @throws org.apache.accumulo.core.client.AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public static void setConnectorInfo(Job job, String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ InputConfigurator.setConnectorInfo(CLASS, job.getConfiguration(), principal, token);
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * Stores the password in a file in HDFS and pulls that into the Distributed Cache in an attempt to be more secure than storing it in the Configuration.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission)
+ * @param tokenFile
+ * the path to the token file
- * @throws AccumuloSecurityException
+ * @since 1.6.0
+ */
+ public static void setConnectorInfo(Job job, String principal, String tokenFile) throws AccumuloSecurityException {
+ InputConfigurator.setConnectorInfo(CLASS, job.getConfiguration(), principal, tokenFile);
+ }
+
+ /**
+ * Determines if the connector has been configured.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the connector has been configured, false otherwise
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static Boolean isConnectorInfoSet(JobContext context) {
+ return InputConfigurator.isConnectorInfoSet(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static String getPrincipal(JobContext context) {
+ return InputConfigurator.getPrincipal(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Gets the serialized token class from either the configuration or the token file.
+ *
+ * @since 1.5.0
+ * @deprecated since 1.6.0; Use {@link #getAuthenticationToken(JobContext)} instead.
+ */
+ @Deprecated
+ protected static String getTokenClass(JobContext context) {
+ return getAuthenticationToken(context).getClass().getName();
+ }
+
+ /**
+ * Gets the serialized token from either the configuration or the token file.
+ *
+ * @since 1.5.0
+ * @deprecated since 1.6.0; Use {@link #getAuthenticationToken(JobContext)} instead.
+ */
+ @Deprecated
+ protected static byte[] getToken(JobContext context) {
+ return AuthenticationToken.AuthenticationTokenSerializer.serialize(getAuthenticationToken(context));
+ }
+
+ /**
+ * Gets the authenticated token from either the specified token file or directly from the configuration, whichever was used when the job was configured.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the principal's authentication token
+ * @since 1.6.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ * @see #setConnectorInfo(Job, String, String)
+ */
+ protected static AuthenticationToken getAuthenticationToken(JobContext context) {
+ return InputConfigurator.getAuthenticationToken(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Configures a {@link org.apache.accumulo.core.client.ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @param zooKeepers
+ * a comma-separated list of zookeeper servers
+ * @since 1.5.0
+ * @deprecated since 1.6.0; Use {@link #setZooKeeperInstance(Job, ClientConfiguration)} instead.
+ */
+ @Deprecated
+ public static void setZooKeeperInstance(Job job, String instanceName, String zooKeepers) {
+ InputConfigurator.setZooKeeperInstance(CLASS, job.getConfiguration(), instanceName, zooKeepers);
+ }
+
+ /**
+ * Configures a {@link org.apache.accumulo.core.client.ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param clientConfig
+ * client configuration containing connection options
+ * @since 1.6.0
+ */
+ public static void setZooKeeperInstance(Job job, ClientConfiguration clientConfig) {
+ InputConfigurator.setZooKeeperInstance(CLASS, job.getConfiguration(), clientConfig);
+ }
+
+ /**
+ * Configures a {@link org.apache.accumulo.core.client.mock.MockInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @since 1.5.0
+ */
+ public static void setMockInstance(Job job, String instanceName) {
+ InputConfigurator.setMockInstance(CLASS, job.getConfiguration(), instanceName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link org.apache.accumulo.core.client.Instance} based on the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return an Accumulo instance
+ * @since 1.5.0
+ * @see #setZooKeeperInstance(Job, String, String)
+ * @see #setMockInstance(Job, String)
+ */
+ protected static Instance getInstance(JobContext context) {
+ return InputConfigurator.getInstance(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param level
+ * the logging level
+ * @since 1.5.0
+ */
+ public static void setLogLevel(Job job, Level level) {
+ InputConfigurator.setLogLevel(CLASS, job.getConfiguration(), level);
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the log level
+ * @since 1.5.0
+ * @see #setLogLevel(Job, Level)
+ */
+ protected static Level getLogLevel(JobContext context) {
+ return InputConfigurator.getLogLevel(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Sets the {@link org.apache.accumulo.core.security.Authorizations} used to scan. Must be a subset of the user's authorization. Defaults to the empty set.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param auths
+ * the user's authorizations
+ */
+ public static void setScanAuthorizations(Job job, Authorizations auths) {
+ InputConfigurator.setScanAuthorizations(CLASS, job.getConfiguration(), auths);
+ }
+
+ /**
+ * Gets the authorizations to set for the scans from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the Accumulo scan authorizations
+ * @since 1.5.0
+ * @see #setScanAuthorizations(Job, Authorizations)
+ */
+ protected static Authorizations getScanAuthorizations(JobContext context) {
+ return InputConfigurator.getScanAuthorizations(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Fetches all {@link InputTableConfig}s that have been set on the given job.
+ *
+ * @param context
+ * the Hadoop job instance to be configured
+ * @return the {@link InputTableConfig} objects for the job
+ * @since 1.6.0
+ */
+ protected static Map<String,InputTableConfig> getInputTableConfigs(JobContext context) {
+ return InputConfigurator.getInputTableConfigs(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Fetches a {@link InputTableConfig} that has been set on the configuration for a specific table.
+ *
+ * <p>
+ * null is returned in the event that the table doesn't exist.
+ *
+ * @param context
+ * the Hadoop job instance to be configured
+ * @param tableName
+ * the table name for which to grab the config object
+ * @return the {@link InputTableConfig} for the given table
+ * @since 1.6.0
+ */
+ protected static InputTableConfig getInputTableConfig(JobContext context, String tableName) {
+ return InputConfigurator.getInputTableConfig(CLASS, getConfiguration(context), tableName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link org.apache.accumulo.core.client.impl.TabletLocator} based on the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @param table
+ * the table for which to initialize the locator
+ * @return an Accumulo tablet locator
+ * @throws org.apache.accumulo.core.client.TableNotFoundException
+ * if the table name set on the configuration doesn't exist
+ * @since 1.6.0
+ */
+ protected static TabletLocator getTabletLocator(JobContext context, String table) throws TableNotFoundException {
+ return InputConfigurator.getTabletLocator(CLASS, getConfiguration(context), table);
+ }
+
+ // InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext job)
+ /**
+ * Check whether a configuration is fully configured to be used with an Accumulo {@link org.apache.hadoop.mapreduce.InputFormat}.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @throws java.io.IOException
+ * if the context is improperly configured
+ * @since 1.5.0
+ */
+ protected static void validateOptions(JobContext context) throws IOException {
+ InputConfigurator.validateOptions(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * An abstract base class to be used to create {@link org.apache.hadoop.mapreduce.RecordReader} instances that convert from Accumulo
+ * {@link org.apache.accumulo.core.data.Key}/{@link org.apache.accumulo.core.data.Value} pairs to the user's K/V types.
+ *
+ * Subclasses must implement {@link #nextKeyValue()} and use it to update the following variables:
+ * <ul>
+ * <li>K {@link #currentK}</li>
+ * <li>V {@link #currentV}</li>
+ * <li>Key {@link #currentKey} (used for progress reporting)</li>
+ * <li>int {@link #numKeysRead} (used for progress reporting)</li>
+ * </ul>
+ */
+ protected abstract static class AbstractRecordReader<K,V> extends RecordReader<K,V> {
+ protected long numKeysRead;
+ protected Iterator<Map.Entry<Key,Value>> scannerIterator;
+ protected RangeInputSplit split;
+
+ /**
+ * Configures the iterators on a scanner for the given table name.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @param scanner
+ * the scanner for which to configure the iterators
+ * @param tableName
+ * the table name for which the scanner is configured
+ * @since 1.6.0
+ */
+ protected abstract void setupIterators(TaskAttemptContext context, Scanner scanner, String tableName, RangeInputSplit split);
+
+ /**
+ * Initialize a scanner over the given input split using this task attempt configuration.
+ */
+ @Override
+ public void initialize(InputSplit inSplit, TaskAttemptContext attempt) throws IOException {
+
+ Scanner scanner;
+ split = (RangeInputSplit) inSplit;
+ log.debug("Initializing input split: " + split.getRange());
+
+ Instance instance = split.getInstance();
+ if (null == instance) {
+ instance = getInstance(attempt);
+ }
+
+ String principal = split.getPrincipal();
+ if (null == principal) {
+ principal = getPrincipal(attempt);
+ }
+
+ AuthenticationToken token = split.getToken();
+ if (null == token) {
+ token = getAuthenticationToken(attempt);
+ }
+
+ Authorizations authorizations = split.getAuths();
+ if (null == authorizations) {
+ authorizations = getScanAuthorizations(attempt);
+ }
+
+ String table = split.getTableName();
+
+ // in case the table name changed, we can still use the previous name for terms of configuration,
+ // but the scanner will use the table id resolved at job setup time
+ InputTableConfig tableConfig = getInputTableConfig(attempt, split.getTableName());
+
+ Boolean isOffline = split.isOffline();
+ if (null == isOffline) {
+ isOffline = tableConfig.isOfflineScan();
+ }
+
+ Boolean isIsolated = split.isIsolatedScan();
+ if (null == isIsolated) {
+ isIsolated = tableConfig.shouldUseIsolatedScanners();
+ }
+
+ Boolean usesLocalIterators = split.usesLocalIterators();
+ if (null == usesLocalIterators) {
+ usesLocalIterators = tableConfig.shouldUseLocalIterators();
+ }
+
+ List<IteratorSetting> iterators = split.getIterators();
+ if (null == iterators) {
+ iterators = tableConfig.getIterators();
+ }
+
+ Collection<Pair<Text,Text>> columns = split.getFetchedColumns();
+ if (null == columns) {
+ columns = tableConfig.getFetchedColumns();
+ }
+
+ try {
+ log.debug("Creating connector with user: " + principal);
+ log.debug("Creating scanner for table: " + table);
+ log.debug("Authorizations are: " + authorizations);
+ if (isOffline) {
+ scanner = new OfflineScanner(instance, new Credentials(principal, token), split.getTableId(), authorizations);
+ } else if (instance instanceof MockInstance) {
+ scanner = instance.getConnector(principal, token).createScanner(split.getTableName(), authorizations);
+ } else {
+ scanner = new ScannerImpl(instance, new Credentials(principal, token), split.getTableId(), authorizations);
+ }
+ if (isIsolated) {
+ log.info("Creating isolated scanner");
+ scanner = new IsolatedScanner(scanner);
+ }
+ if (usesLocalIterators) {
+ log.info("Using local iterators");
+ scanner = new ClientSideIteratorScanner(scanner);
+ }
+
+ setupIterators(attempt, scanner, split.getTableName(), split);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ // setup a scanner within the bounds of this split
+ for (Pair<Text,Text> c : columns) {
+ if (c.getSecond() != null) {
+ log.debug("Fetching column " + c.getFirst() + ":" + c.getSecond());
+ scanner.fetchColumn(c.getFirst(), c.getSecond());
+ } else {
+ log.debug("Fetching column family " + c.getFirst());
+ scanner.fetchColumnFamily(c.getFirst());
+ }
+ }
+
+ scanner.setRange(split.getRange());
+ numKeysRead = 0;
+
+ // do this last after setting all scanner options
+ scannerIterator = scanner.iterator();
+ }
+
+ @Override
+ public void close() {}
+
+ @Override
+ public float getProgress() throws IOException {
+ if (numKeysRead > 0 && currentKey == null)
+ return 1.0f;
+ return split.getProgress(currentKey);
+ }
+
+ /**
+ * The Key that should be returned to the client
+ */
+ protected K currentK = null;
+
+ /**
+ * The Value that should be return to the client
+ */
+ protected V currentV = null;
+
+ /**
+ * The Key that is used to determine progress in the current InputSplit. It is not returned to the client and is only used internally
+ */
+ protected Key currentKey = null;
+
+ @Override
+ public K getCurrentKey() throws IOException, InterruptedException {
+ return currentK;
+ }
+
+ @Override
+ public V getCurrentValue() throws IOException, InterruptedException {
+ return currentV;
+ }
+ }
+
+ Map<String,Map<KeyExtent,List<Range>>> binOfflineTable(JobContext context, String tableId, List<Range> ranges) throws TableNotFoundException,
+ AccumuloException, AccumuloSecurityException {
+
+ Instance instance = getInstance(context);
+ Connector conn = instance.getConnector(getPrincipal(context), getAuthenticationToken(context));
+
+ return InputConfigurator.binOffline(tableId, ranges, instance, conn);
+ }
+
+ /**
+ * Gets the splits of the tables that have been set on the job.
+ *
+ * @param context
+ * the configuration of the job
+ * @return the splits from the tables based on the ranges.
+ * @throws java.io.IOException
+ * if a table set on the job doesn't exist or an error occurs initializing the tablet locator
+ */
++ @Override
+ public List<InputSplit> getSplits(JobContext context) throws IOException {
+ Level logLevel = getLogLevel(context);
+ log.setLevel(logLevel);
+ validateOptions(context);
+ Random random = new Random();
+ LinkedList<InputSplit> splits = new LinkedList<InputSplit>();
+ Map<String,InputTableConfig> tableConfigs = getInputTableConfigs(context);
+ for (Map.Entry<String,InputTableConfig> tableConfigEntry : tableConfigs.entrySet()) {
+
+ String tableName = tableConfigEntry.getKey();
+ InputTableConfig tableConfig = tableConfigEntry.getValue();
+
+ Instance instance = getInstance(context);
+ boolean mockInstance;
+ String tableId;
+ // resolve table name to id once, and use id from this point forward
+ if (instance instanceof MockInstance) {
+ tableId = "";
+ mockInstance = true;
+ } else {
+ try {
+ tableId = Tables.getTableId(instance, tableName);
+ } catch (TableNotFoundException e) {
+ throw new IOException(e);
+ }
+ mockInstance = false;
+ }
+
+ Authorizations auths = getScanAuthorizations(context);
+ String principal = getPrincipal(context);
+ AuthenticationToken token = getAuthenticationToken(context);
+
+ boolean autoAdjust = tableConfig.shouldAutoAdjustRanges();
+ List<Range> ranges = autoAdjust ? Range.mergeOverlapping(tableConfig.getRanges()) : tableConfig.getRanges();
+ if (ranges.isEmpty()) {
+ ranges = new ArrayList<Range>(1);
+ ranges.add(new Range());
+ }
+
+ // get the metadata information for these ranges
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+ TabletLocator tl;
+ try {
+ if (tableConfig.isOfflineScan()) {
+ binnedRanges = binOfflineTable(context, tableId, ranges);
+ while (binnedRanges == null) {
+ // Some tablets were still online, try again
+ UtilWaitThread.sleep(100 + random.nextInt(100)); // sleep randomly between 100 and 200 ms
+ binnedRanges = binOfflineTable(context, tableId, ranges);
+
+ }
+ } else {
+ tl = getTabletLocator(context, tableId);
+ // its possible that the cache could contain complete, but old information about a tables tablets... so clear it
+ tl.invalidateCache();
+ Credentials creds = new Credentials(getPrincipal(context), getAuthenticationToken(context));
+
+ while (!tl.binRanges(creds, ranges, binnedRanges).isEmpty()) {
+ if (!(instance instanceof MockInstance)) {
+ if (!Tables.exists(instance, tableId))
+ throw new TableDeletedException(tableId);
+ if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
+ throw new TableOfflineException(instance, tableId);
+ }
+ binnedRanges.clear();
+ log.warn("Unable to locate bins for specified ranges. Retrying.");
+ UtilWaitThread.sleep(100 + random.nextInt(100)); // sleep randomly between 100 and 200 ms
+ tl.invalidateCache();
+ }
+ }
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ HashMap<Range,ArrayList<String>> splitsToAdd = null;
+
+ if (!autoAdjust)
+ splitsToAdd = new HashMap<Range,ArrayList<String>>();
+
+ HashMap<String,String> hostNameCache = new HashMap<String,String>();
+ for (Map.Entry<String,Map<KeyExtent,List<Range>>> tserverBin : binnedRanges.entrySet()) {
+ String ip = tserverBin.getKey().split(":", 2)[0];
+ String location = hostNameCache.get(ip);
+ if (location == null) {
+ InetAddress inetAddress = InetAddress.getByName(ip);
+ location = inetAddress.getCanonicalHostName();
+ hostNameCache.put(ip, location);
+ }
+ for (Map.Entry<KeyExtent,List<Range>> extentRanges : tserverBin.getValue().entrySet()) {
+ Range ke = extentRanges.getKey().toDataRange();
+ for (Range r : extentRanges.getValue()) {
+ if (autoAdjust) {
+ // divide ranges into smaller ranges, based on the tablets
+ RangeInputSplit split = new RangeInputSplit(tableName, tableId, ke.clip(r), new String[] {location});
+
+ split.setOffline(tableConfig.isOfflineScan());
+ split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
+ split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());
+ split.setMockInstance(mockInstance);
+ split.setFetchedColumns(tableConfig.getFetchedColumns());
+ split.setPrincipal(principal);
+ split.setToken(token);
+ split.setInstanceName(instance.getInstanceName());
+ split.setZooKeepers(instance.getZooKeepers());
+ split.setAuths(auths);
+ split.setIterators(tableConfig.getIterators());
+ split.setLogLevel(logLevel);
+
+ splits.add(split);
+ } else {
+ // don't divide ranges
+ ArrayList<String> locations = splitsToAdd.get(r);
+ if (locations == null)
+ locations = new ArrayList<String>(1);
+ locations.add(location);
+ splitsToAdd.put(r, locations);
+ }
+ }
+ }
+ }
+
+ if (!autoAdjust)
+ for (Map.Entry<Range,ArrayList<String>> entry : splitsToAdd.entrySet()) {
+ RangeInputSplit split = new RangeInputSplit(tableName, tableId, entry.getKey(), entry.getValue().toArray(new String[0]));
+
+ split.setOffline(tableConfig.isOfflineScan());
+ split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
+ split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());
+ split.setMockInstance(mockInstance);
+ split.setFetchedColumns(tableConfig.getFetchedColumns());
+ split.setPrincipal(principal);
+ split.setToken(token);
+ split.setInstanceName(instance.getInstanceName());
+ split.setZooKeepers(instance.getZooKeepers());
+ split.setAuths(auths);
+ split.setIterators(tableConfig.getIterators());
+ split.setLogLevel(logLevel);
+
+ splits.add(split);
+ }
+ }
+ return splits;
+ }
+
+ // use reflection to pull the Configuration out of the JobContext for Hadoop 1 and Hadoop 2 compatibility
+ static Configuration getConfiguration(JobContext context) {
+ try {
+ Class<?> c = AbstractInputFormat.class.getClassLoader().loadClass("org.apache.hadoop.mapreduce.JobContext");
+ Method m = c.getMethod("getConfiguration");
+ Object o = m.invoke(context, new Object[0]);
+ return (Configuration) o;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
index 0c924b1,1f83541..2c01b0d
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
@@@ -92,26 -92,7 +91,25 @@@ public class AccumuloOutputFormat exten
public static void setConnectorInfo(Job job, String principal, AuthenticationToken token) throws AccumuloSecurityException {
OutputConfigurator.setConnectorInfo(CLASS, job.getConfiguration(), principal, token);
}
-
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * Stores the password in a file in HDFS and pulls that into the Distributed Cache in an attempt to be more secure than storing it in the Configuration.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission if {@link #setCreateTables(Job, boolean)} is set to true)
+ * @param tokenFile
+ * the path to the token file
- * @throws AccumuloSecurityException
+ * @since 1.6.0
+ */
+ public static void setConnectorInfo(Job job, String principal, String tokenFile) throws AccumuloSecurityException {
+ OutputConfigurator.setConnectorInfo(CLASS, job.getConfiguration(), principal, tokenFile);
+ }
+
/**
* Determines if the connector has been configured.
*
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputTableConfig.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputTableConfig.java
index 808bd7c,0000000..e59451e
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputTableConfig.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputTableConfig.java
@@@ -1,370 -1,0 +1,367 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * This class to holds a batch scan configuration for a table. It contains all the properties needed to specify how rows should be returned from the table.
+ */
+public class InputTableConfig implements Writable {
+
+ private List<IteratorSetting> iterators;
+ private List<Range> ranges;
+ private Collection<Pair<Text,Text>> columns;
+
+ private boolean autoAdjustRanges = true;
+ private boolean useLocalIterators = false;
+ private boolean useIsolatedScanners = false;
+ private boolean offlineScan = false;
+
+ public InputTableConfig() {}
+
+ /**
+ * Creates a batch scan config object out of a previously serialized batch scan config object.
+ *
+ * @param input
+ * the data input of the serialized batch scan config
- * @throws IOException
+ */
+ public InputTableConfig(DataInput input) throws IOException {
+ readFields(input);
+ }
+
+ /**
+ * Sets the input ranges to scan for all tables associated with this job. This will be added to any per-table ranges that have been set using
+ *
+ * @param ranges
+ * the ranges that will be mapped over
+ * @since 1.6.0
+ */
+ public InputTableConfig setRanges(List<Range> ranges) {
+ this.ranges = ranges;
+ return this;
+ }
+
+ /**
+ * Returns the ranges to be queried in the configuration
+ */
+ public List<Range> getRanges() {
+ return ranges != null ? ranges : new ArrayList<Range>();
+ }
+
+ /**
+ * Restricts the columns that will be mapped over for this job for the default input table.
+ *
+ * @param columns
+ * a pair of {@link Text} objects corresponding to column family and column qualifier. If the column qualifier is null, the entire column family is
+ * selected. An empty set is the default and is equivalent to scanning the all columns.
+ * @since 1.6.0
+ */
+ public InputTableConfig fetchColumns(Collection<Pair<Text,Text>> columns) {
+ this.columns = columns;
+ return this;
+ }
+
+ /**
+ * Returns the columns to be fetched for this configuration
+ */
+ public Collection<Pair<Text,Text>> getFetchedColumns() {
+ return columns != null ? columns : new HashSet<Pair<Text,Text>>();
+ }
+
+ /**
+ * Set iterators on to be used in the query.
+ *
+ * @param iterators
+ * the configurations for the iterators
+ * @since 1.6.0
+ */
+ public InputTableConfig setIterators(List<IteratorSetting> iterators) {
+ this.iterators = iterators;
+ return this;
+ }
+
+ /**
+ * Returns the iterators to be set on this configuration
+ */
+ public List<IteratorSetting> getIterators() {
+ return iterators != null ? iterators : new ArrayList<IteratorSetting>();
+ }
+
+ /**
+ * Controls the automatic adjustment of ranges for this job. This feature merges overlapping ranges, then splits them to align with tablet boundaries.
+ * Disabling this feature will cause exactly one Map task to be created for each specified range. The default setting is enabled. *
+ *
+ * <p>
+ * By default, this feature is <b>enabled</b>.
+ *
+ * @param autoAdjustRanges
+ * the feature is enabled if true, disabled otherwise
+ * @see #setRanges(java.util.List)
+ * @since 1.6.0
+ */
+ public InputTableConfig setAutoAdjustRanges(boolean autoAdjustRanges) {
+ this.autoAdjustRanges = autoAdjustRanges;
+ return this;
+ }
+
+ /**
+ * Determines whether a configuration has auto-adjust ranges enabled.
+ *
+ * @return false if the feature is disabled, true otherwise
+ * @since 1.6.0
+ * @see #setAutoAdjustRanges(boolean)
+ */
+ public boolean shouldAutoAdjustRanges() {
+ return autoAdjustRanges;
+ }
+
+ /**
+ * Controls the use of the {@link org.apache.accumulo.core.client.ClientSideIteratorScanner} in this job. Enabling this feature will cause the iterator stack
+ * to be constructed within the Map task, rather than within the Accumulo TServer. To use this feature, all classes needed for those iterators must be
+ * available on the classpath for the task.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param useLocalIterators
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public InputTableConfig setUseLocalIterators(boolean useLocalIterators) {
+ this.useLocalIterators = useLocalIterators;
+ return this;
+ }
+
+ /**
+ * Determines whether a configuration uses local iterators.
+ *
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setUseLocalIterators(boolean)
+ */
+ public boolean shouldUseLocalIterators() {
+ return useLocalIterators;
+ }
+
+ /**
+ * <p>
+ * Enable reading offline tables. By default, this feature is disabled and only online tables are scanned. This will make the map reduce job directly read the
+ * table's files. If the table is not offline, then the job will fail. If the table comes online during the map reduce job, it is likely that the job will
+ * fail.
+ *
+ * <p>
+ * To use this option, the map reduce user will need access to read the Accumulo directory in HDFS.
+ *
+ * <p>
+ * Reading the offline table will create the scan time iterator stack in the map process. So any iterators that are configured for the table will need to be
+ * on the mapper's classpath. The accumulo-site.xml may need to be on the mapper's classpath if HDFS or the Accumulo directory in HDFS are non-standard.
+ *
+ * <p>
+ * One way to use this feature is to clone a table, take the clone offline, and use the clone as the input table for a map reduce job. If you plan to map
+ * reduce over the data many times, it may be better to the compact the table, clone it, take it offline, and use the clone for all map reduce jobs. The
+ * reason to do this is that compaction will reduce each tablet in the table to one file, and it is faster to read from one file.
+ *
+ * <p>
+ * There are two possible advantages to reading a tables file directly out of HDFS. First, you may see better read performance. Second, it will support
+ * speculative execution better. When reading an online table speculative execution can put more load on an already slow tablet server.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param offlineScan
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public InputTableConfig setOfflineScan(boolean offlineScan) {
+ this.offlineScan = offlineScan;
+ return this;
+ }
+
+ /**
+ * Determines whether a configuration has the offline table scan feature enabled.
+ *
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setOfflineScan(boolean)
+ */
+ public boolean isOfflineScan() {
+ return offlineScan;
+ }
+
+ /**
+ * Controls the use of the {@link org.apache.accumulo.core.client.IsolatedScanner} in this job.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param useIsolatedScanners
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public InputTableConfig setUseIsolatedScanners(boolean useIsolatedScanners) {
+ this.useIsolatedScanners = useIsolatedScanners;
+ return this;
+ }
+
+ /**
+ * Determines whether a configuration has isolation enabled.
+ *
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setUseIsolatedScanners(boolean)
+ */
+ public boolean shouldUseIsolatedScanners() {
+ return useIsolatedScanners;
+ }
+
+ /**
+ * Writes the state for the current object out to the specified {@link DataOutput}
+ *
+ * @param dataOutput
+ * the output for which to write the object's state
- * @throws IOException
+ */
+ @Override
+ public void write(DataOutput dataOutput) throws IOException {
+ if (iterators != null) {
+ dataOutput.writeInt(iterators.size());
+ for (IteratorSetting setting : iterators)
+ setting.write(dataOutput);
+ } else {
+ dataOutput.writeInt(0);
+ }
+ if (ranges != null) {
+ dataOutput.writeInt(ranges.size());
+ for (Range range : ranges)
+ range.write(dataOutput);
+ } else {
+ dataOutput.writeInt(0);
+ }
+ if (columns != null) {
+ dataOutput.writeInt(columns.size());
+ for (Pair<Text,Text> column : columns) {
+ if (column.getSecond() == null) {
+ dataOutput.writeInt(1);
+ column.getFirst().write(dataOutput);
+ } else {
+ dataOutput.writeInt(2);
+ column.getFirst().write(dataOutput);
+ column.getSecond().write(dataOutput);
+ }
+ }
+ } else {
+ dataOutput.writeInt(0);
+ }
+ dataOutput.writeBoolean(autoAdjustRanges);
+ dataOutput.writeBoolean(useLocalIterators);
+ dataOutput.writeBoolean(useIsolatedScanners);
+ }
+
+ /**
+ * Reads the fields in the {@link DataInput} into the current object
+ *
+ * @param dataInput
+ * the input fields to read into the current object
- * @throws IOException
+ */
+ @Override
+ public void readFields(DataInput dataInput) throws IOException {
+ // load iterators
+ long iterSize = dataInput.readInt();
+ if (iterSize > 0)
+ iterators = new ArrayList<IteratorSetting>();
+ for (int i = 0; i < iterSize; i++)
+ iterators.add(new IteratorSetting(dataInput));
+ // load ranges
+ long rangeSize = dataInput.readInt();
+ if (rangeSize > 0)
+ ranges = new ArrayList<Range>();
+ for (int i = 0; i < rangeSize; i++) {
+ Range range = new Range();
+ range.readFields(dataInput);
+ ranges.add(range);
+ }
+ // load columns
+ long columnSize = dataInput.readInt();
+ if (columnSize > 0)
+ columns = new HashSet<Pair<Text,Text>>();
+ for (int i = 0; i < columnSize; i++) {
+ long numPairs = dataInput.readInt();
+ Text colFam = new Text();
+ colFam.readFields(dataInput);
+ if (numPairs == 1) {
+ columns.add(new Pair<Text,Text>(colFam, null));
+ } else if (numPairs == 2) {
+ Text colQual = new Text();
+ colQual.readFields(dataInput);
+ columns.add(new Pair<Text,Text>(colFam, colQual));
+ }
+ }
+ autoAdjustRanges = dataInput.readBoolean();
+ useLocalIterators = dataInput.readBoolean();
+ useIsolatedScanners = dataInput.readBoolean();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ InputTableConfig that = (InputTableConfig) o;
+
+ if (autoAdjustRanges != that.autoAdjustRanges)
+ return false;
+ if (offlineScan != that.offlineScan)
+ return false;
+ if (useIsolatedScanners != that.useIsolatedScanners)
+ return false;
+ if (useLocalIterators != that.useLocalIterators)
+ return false;
+ if (columns != null ? !columns.equals(that.columns) : that.columns != null)
+ return false;
+ if (iterators != null ? !iterators.equals(that.iterators) : that.iterators != null)
+ return false;
+ if (ranges != null ? !ranges.equals(that.ranges) : that.ranges != null)
+ return false;
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = 31 * (iterators != null ? iterators.hashCode() : 0);
+ result = 31 * result + (ranges != null ? ranges.hashCode() : 0);
+ result = 31 * result + (columns != null ? columns.hashCode() : 0);
+ result = 31 * result + (autoAdjustRanges ? 1 : 0);
+ result = 31 * result + (useLocalIterators ? 1 : 0);
+ result = 31 * result + (useIsolatedScanners ? 1 : 0);
+ result = 31 * result + (offlineScan ? 1 : 0);
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
index cf861ce,1a029dc..54ff976
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
@@@ -125,43 -102,8 +124,42 @@@ public class ConfiguratorBase
ArgumentChecker.notNull(principal, token);
conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
conf.set(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL), principal);
- conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN_CLASS), token.getClass().getCanonicalName());
- conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN), CredentialHelper.tokenAsBase64(token));
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN),
+ TokenSource.INLINE.prefix() + token.getClass().getName() + ":" + Base64.encodeBase64String(AuthenticationTokenSerializer.serialize(token)));
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * Pulls a token file into the Distributed Cache that contains the authentication token in an attempt to be more secure than storing the password in the
+ * Configuration. Token file created with "bin/accumulo create-token".
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param principal
+ * a valid Accumulo user name
+ * @param tokenFile
+ * the path to the token file in DFS
- * @throws AccumuloSecurityException
+ * @since 1.6.0
+ */
+ public static void setConnectorInfo(Class<?> implementingClass, Configuration conf, String principal, String tokenFile) throws AccumuloSecurityException {
+ if (isConnectorInfoSet(implementingClass, conf))
+ throw new IllegalStateException("Connector info for " + implementingClass.getSimpleName() + " can only be set once per job");
+
+ ArgumentChecker.notNull(principal, tokenFile);
+
+ try {
+ DistributedCacheHelper.addCacheFile(new URI(tokenFile), conf);
+ } catch (URISyntaxException e) {
+ throw new IllegalStateException("Unable to add tokenFile \"" + tokenFile + "\" to distributed cache.");
+ }
+
+ conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL), principal);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN), TokenSource.FILE.prefix() + tokenFile);
}
/**
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/data/Condition.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Condition.java
index c80dcd6,0000000..bfd4818
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/data/Condition.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Condition.java
@@@ -1,245 -1,0 +1,238 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
+import java.util.Arrays;
+import java.util.HashSet;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Conditions that must be met on a particular column in a row.
+ *
+ * @since 1.6.0
+ */
+public class Condition {
+
+ private ByteSequence cf;
+ private ByteSequence cq;
+ private ByteSequence cv;
+ private ByteSequence val;
+ private Long ts;
+ private IteratorSetting iterators[] = new IteratorSetting[0];
+ private static final ByteSequence EMPTY = new ArrayByteSequence(new byte[0]);
+
+
+ public Condition(CharSequence cf, CharSequence cq) {
+ ArgumentChecker.notNull(cf, cq);
+ this.cf = new ArrayByteSequence(cf.toString().getBytes(Constants.UTF8));
+ this.cq = new ArrayByteSequence(cq.toString().getBytes(Constants.UTF8));
+ this.cv = EMPTY;
+ }
+
+ public Condition(byte[] cf, byte[] cq) {
+ ArgumentChecker.notNull(cf, cq);
+ this.cf = new ArrayByteSequence(cf);
+ this.cq = new ArrayByteSequence(cq);
+ this.cv = EMPTY;
+ }
+
+ public Condition(Text cf, Text cq) {
+ ArgumentChecker.notNull(cf, cq);
+ this.cf = new ArrayByteSequence(cf.getBytes(), 0, cf.getLength());
+ this.cq = new ArrayByteSequence(cq.getBytes(), 0, cq.getLength());
+ this.cv = EMPTY;
+ }
+
+ public Condition(ByteSequence cf, ByteSequence cq) {
+ ArgumentChecker.notNull(cf, cq);
+ this.cf = cf;
+ this.cq = cq;
+ this.cv = EMPTY;
+ }
+
+ public ByteSequence getFamily() {
+ return cf;
+ }
+
+ public ByteSequence getQualifier() {
+ return cq;
+ }
+
+ /**
+ * Sets the version for the column to check. If this is not set then the latest column will be checked, unless iterators do something different.
+ *
- * @param ts
+ * @return returns this
+ */
+
+ public Condition setTimestamp(long ts) {
+ this.ts = ts;
+ return this;
+ }
+
+ public Long getTimestamp() {
+ return ts;
+ }
+
+ /**
+ * see {@link #setValue(byte[])}
+ *
- * @param value
+ * @return returns this
+ */
+
+ public Condition setValue(CharSequence value) {
+ ArgumentChecker.notNull(value);
+ this.val = new ArrayByteSequence(value.toString().getBytes(Constants.UTF8));
+ return this;
+ }
+
+ /**
+ * This method sets the expected value of a column. Inorder for the condition to pass the column must exist and have this value. If a value is not set, then
+ * the column must be absent for the condition to pass.
+ *
- * @param value
+ * @return returns this
+ */
+
+ public Condition setValue(byte[] value) {
+ ArgumentChecker.notNull(value);
+ this.val = new ArrayByteSequence(value);
+ return this;
+ }
+
+ /**
+ * see {@link #setValue(byte[])}
+ *
- * @param value
+ * @return returns this
+ */
+
+ public Condition setValue(Text value) {
+ ArgumentChecker.notNull(value);
+ this.val = new ArrayByteSequence(value.getBytes(), 0, value.getLength());
+ return this;
+ }
+
+ /**
+ * see {@link #setValue(byte[])}
+ *
- * @param value
+ * @return returns this
+ */
+
+ public Condition setValue(ByteSequence value) {
+ ArgumentChecker.notNull(value);
+ this.val = value;
+ return this;
+ }
+
+ public ByteSequence getValue() {
+ return val;
+ }
+
+ /**
+ * Sets the visibility for the column to check. If not set it defaults to empty visibility.
+ *
- * @param cv
+ * @return returns this
+ */
+
+ public Condition setVisibility(ColumnVisibility cv) {
+ ArgumentChecker.notNull(cv);
+ this.cv = new ArrayByteSequence(cv.getExpression());
+ return this;
+ }
+
+ public ByteSequence getVisibility() {
+ return cv;
+ }
+
+ /**
+ * Set iterators to use when reading the columns value. These iterators will be applied in addition to the iterators configured for the table. Using iterators
+ * its possible to test other conditions, besides equality and absence, like less than. On the server side the iterators will be seeked using a range that
+ * covers only the family, qualifier, and visibility (if the timestamp is set then it will be used to narrow the range). Value equality will be tested using
+ * the first entry returned by the iterator stack.
+ *
- * @param iterators
+ * @return returns this
+ */
+
+ public Condition setIterators(IteratorSetting... iterators) {
+ ArgumentChecker.notNull(iterators);
+
+ if (iterators.length > 1) {
+ HashSet<String> names = new HashSet<String>();
+ HashSet<Integer> prios = new HashSet<Integer>();
+
+ for (IteratorSetting iteratorSetting : iterators) {
+ if (!names.add(iteratorSetting.getName()))
+ throw new IllegalArgumentException("iterator name used more than once " + iteratorSetting.getName());
+ if (!prios.add(iteratorSetting.getPriority()))
+ throw new IllegalArgumentException("iterator priority used more than once " + iteratorSetting.getPriority());
+ }
+ }
+
+ this.iterators = iterators;
+ return this;
+ }
+
+ public IteratorSetting[] getIterators() {
+ return iterators;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this) {
+ return true;
+ }
+ if (o == null || !(o instanceof Condition)) {
+ return false;
+ }
+ Condition c = (Condition) o;
+ if (!(c.cf.equals(cf))) {
+ return false;
+ }
+ if (!(c.cq.equals(cq))) {
+ return false;
+ }
+ if (!(c.cv.equals(cv))) {
+ return false;
+ }
+ if (!(c.val == null ? val == null : c.val.equals(val))) {
+ return false;
+ }
+ if (!(c.ts == null ? ts == null : c.ts.equals(ts))) {
+ return false;
+ }
+ if (!(Arrays.equals(c.iterators, iterators))) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = 17;
+ result = 31 * result + cf.hashCode();
+ result = 31 * result + cq.hashCode();
+ result = 31 * result + cv.hashCode();
+ result = 31 * result + (val == null ? 0 : val.hashCode());
+ result = 31 * result + (ts == null ? 0 : ts.hashCode());
+ result = 31 * result + Arrays.hashCode(iterators);
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/data/Range.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index 6c3ea0d,7d15851..ca97e01
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@@ -136,13 -113,8 +136,11 @@@ public final class BCFile
/**
* @param compressionAlgo
* The compression algorithm to be used to for compression.
+ * @param cryptoModule
+ * the module to use to obtain cryptographic streams
- * @param cryptoParams
- * @throws IOException
*/
- public WBlockState(Algorithm compressionAlgo, FSDataOutputStream fsOut, BytesWritable fsOutputBuffer, Configuration conf) throws IOException {
+ public WBlockState(Algorithm compressionAlgo, FSDataOutputStream fsOut, BytesWritable fsOutputBuffer, Configuration conf, CryptoModule cryptoModule,
+ CryptoModuleParameters cryptoParams) throws IOException {
this.compressAlgo = compressionAlgo;
this.fsOut = fsOut;
this.posStart = fsOut.getPos();
@@@ -343,10 -263,9 +339,9 @@@
* FS output stream.
* @param compressionName
* Name of the compression algorithm, which will be used for all data blocks.
- * @throws IOException
* @see Compression#getSupportedAlgorithms
*/
- public Writer(FSDataOutputStream fout, String compressionName, Configuration conf, boolean trackDataBlocks) throws IOException {
+ public Writer(FSDataOutputStream fout, String compressionName, Configuration conf, boolean trackDataBlocks, AccumuloConfiguration accumuloConfiguration) throws IOException {
if (fout.getPos() != 0) {
throw new IOException("Output file not at zero offset.");
}
@@@ -507,7 -402,8 +500,8 @@@
this.name = name;
this.compressAlgo = compressAlgo;
}
-
+
+ @Override
public void register(long raw, long begin, long end) {
metaIndex.addEntry(new MetaIndexEntry(name, compressAlgo, new BlockRegion(begin, end - begin, raw)));
}
@@@ -521,7 -417,8 +515,8 @@@
DataBlockRegister() {
// do nothing
}
-
+
+ @Override
public void register(long raw, long begin, long end) {
dataIndex.addBlockRegion(new BlockRegion(begin, end - begin, raw));
}
@@@ -734,37 -560,22 +729,36 @@@
* FS input stream.
* @param fileLength
* Length of the corresponding file
- * @throws IOException
*/
- public Reader(FSDataInputStream fin, long fileLength, Configuration conf) throws IOException {
+ public Reader(FSDataInputStream fin, long fileLength, Configuration conf, AccumuloConfiguration accumuloConfiguration) throws IOException {
+
this.in = fin;
this.conf = conf;
-
- // move the cursor to the beginning of the tail, containing: offset to the
- // meta block index, version and magic
- fin.seek(fileLength - Magic.size() - Version.size() - Long.SIZE / Byte.SIZE);
- long offsetIndexMeta = fin.readLong();
+
+ // Move the cursor to grab the version and the magic first
+ fin.seek(fileLength - Magic.size() - Version.size());
version = new Version(fin);
Magic.readAndVerify(fin);
-
- if (!version.compatibleWith(BCFile.API_VERSION)) {
+
+ // Do a version check
+ if (!version.compatibleWith(BCFile.API_VERSION) && !version.equals(BCFile.API_VERSION_1)) {
throw new RuntimeException("Incompatible BCFile fileBCFileVersion.");
}
-
+
+ // Read the right number offsets based on version
+ long offsetIndexMeta = 0;
+ long offsetCryptoParameters = 0;
+
+ if (version.equals(API_VERSION_1)) {
+ fin.seek(fileLength - Magic.size() - Version.size() - (Long.SIZE / Byte.SIZE));
+ offsetIndexMeta = fin.readLong();
+
+ } else {
+ fin.seek(fileLength - Magic.size() - Version.size() - (2 * (Long.SIZE / Byte.SIZE)));
+ offsetIndexMeta = fin.readLong();
+ offsetCryptoParameters = fin.readLong();
+ }
+
// read meta index
fin.seek(offsetIndexMeta);
metaIndex = new MetaIndex(fin);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
index 541a379,53ea0e8..0ebf4d8
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
@@@ -628,11 -628,11 +628,11 @@@ abstract public class TransformingItera
* @return the part of the key this iterator is not transforming
*/
abstract protected PartialKey getKeyPrefix();
-
- public static interface KVBuffer {
+
+ public interface KVBuffer {
void append(Key key, Value val);
}
-
+
/**
* Transforms {@code input}. This method must not change the row part of the key, and must only change the parts of the key after the return value of
* {@link #getKeyPrefix()}. Implementors must also remember to copy the delete flag from {@code originalKey} onto the new key. Or, implementors should use one
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/security/SecurityUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleFactory.java
----------------------------------------------------------------------
[09/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFileDumper.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFileDumper.java
index 9b9cd51,0000000..4b3e41c
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFileDumper.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFileDumper.java
@@@ -1,259 -1,0 +1,258 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.accumulo.core.file.rfile.bcfile;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.file.rfile.bcfile.BCFile.BlockRegion;
+import org.apache.accumulo.core.file.rfile.bcfile.BCFile.MetaIndexEntry;
+import org.apache.accumulo.core.file.rfile.bcfile.TFile.TFileIndexEntry;
+import org.apache.accumulo.core.file.rfile.bcfile.Utils.Version;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+
+/**
+ * Dumping the information of a TFile.
+ */
+class TFileDumper {
+ static final Log LOG = LogFactory.getLog(TFileDumper.class);
+
+ private TFileDumper() {
+ // namespace object not constructable.
+ }
+
+ private enum Align {
+ LEFT, CENTER, RIGHT, ZERO_PADDED;
+ static String format(String s, int width, Align align) {
+ if (s.length() >= width)
+ return s;
+ int room = width - s.length();
+ Align alignAdjusted = align;
+ if (room == 1) {
+ alignAdjusted = LEFT;
+ }
+ if (alignAdjusted == LEFT) {
+ return s + String.format("%" + room + "s", "");
+ }
+ if (alignAdjusted == RIGHT) {
+ return String.format("%" + room + "s", "") + s;
+ }
+ if (alignAdjusted == CENTER) {
+ int half = room / 2;
+ return String.format("%" + half + "s", "") + s + String.format("%" + (room - half) + "s", "");
+ }
+ throw new IllegalArgumentException("Unsupported alignment");
+ }
+
+ static String format(long l, int width, Align align) {
+ if (align == ZERO_PADDED) {
+ return String.format("%0" + width + "d", l);
+ }
+ return format(Long.toString(l), width, align);
+ }
+
+ static int calculateWidth(String caption, long max) {
+ return Math.max(caption.length(), Long.toString(max).length());
+ }
+ }
+
+ /**
+ * Dump information about TFile.
+ *
+ * @param file
+ * Path string of the TFile
+ * @param out
+ * PrintStream to output the information.
+ * @param conf
+ * The configuration object.
- * @throws IOException
+ */
+ static public void dumpInfo(String file, PrintStream out, Configuration conf) throws IOException {
+ final int maxKeySampleLen = 16;
+ Path path = new Path(file);
+ FileSystem fs = path.getFileSystem(conf);
+ long length = fs.getFileStatus(path).getLen();
+ FSDataInputStream fsdis = fs.open(path);
+ TFile.Reader reader = new TFile.Reader(fsdis, length, conf);
+ try {
+ LinkedHashMap<String,String> properties = new LinkedHashMap<String,String>();
+ int blockCnt = reader.readerBCF.getBlockCount();
+ int metaBlkCnt = reader.readerBCF.metaIndex.index.size();
+ properties.put("BCFile Version", reader.readerBCF.version.toString());
+ properties.put("TFile Version", reader.tfileMeta.version.toString());
+ properties.put("File Length", Long.toString(length));
+ properties.put("Data Compression", reader.readerBCF.getDefaultCompressionName());
+ properties.put("Record Count", Long.toString(reader.getEntryCount()));
+ properties.put("Sorted", Boolean.toString(reader.isSorted()));
+ if (reader.isSorted()) {
+ properties.put("Comparator", reader.getComparatorName());
+ }
+ properties.put("Data Block Count", Integer.toString(blockCnt));
+ long dataSize = 0, dataSizeUncompressed = 0;
+ if (blockCnt > 0) {
+ for (int i = 0; i < blockCnt; ++i) {
+ BlockRegion region = reader.readerBCF.dataIndex.getBlockRegionList().get(i);
+ dataSize += region.getCompressedSize();
+ dataSizeUncompressed += region.getRawSize();
+ }
+ properties.put("Data Block Bytes", Long.toString(dataSize));
+ if (reader.readerBCF.getDefaultCompressionName() != "none") {
+ properties.put("Data Block Uncompressed Bytes", Long.toString(dataSizeUncompressed));
+ properties.put("Data Block Compression Ratio", String.format("1:%.1f", (double) dataSizeUncompressed / dataSize));
+ }
+ }
+
+ properties.put("Meta Block Count", Integer.toString(metaBlkCnt));
+ long metaSize = 0, metaSizeUncompressed = 0;
+ if (metaBlkCnt > 0) {
+ Collection<MetaIndexEntry> metaBlks = reader.readerBCF.metaIndex.index.values();
+ boolean calculateCompression = false;
+ for (Iterator<MetaIndexEntry> it = metaBlks.iterator(); it.hasNext();) {
+ MetaIndexEntry e = it.next();
+ metaSize += e.getRegion().getCompressedSize();
+ metaSizeUncompressed += e.getRegion().getRawSize();
+ if (e.getCompressionAlgorithm() != Compression.Algorithm.NONE) {
+ calculateCompression = true;
+ }
+ }
+ properties.put("Meta Block Bytes", Long.toString(metaSize));
+ if (calculateCompression) {
+ properties.put("Meta Block Uncompressed Bytes", Long.toString(metaSizeUncompressed));
+ properties.put("Meta Block Compression Ratio", String.format("1:%.1f", (double) metaSizeUncompressed / metaSize));
+ }
+ }
+ properties.put("Meta-Data Size Ratio", String.format("1:%.1f", (double) dataSize / metaSize));
+ long leftOverBytes = length - dataSize - metaSize;
+ long miscSize = BCFile.Magic.size() * 2 + Long.SIZE / Byte.SIZE + Version.size();
+ long metaIndexSize = leftOverBytes - miscSize;
+ properties.put("Meta Block Index Bytes", Long.toString(metaIndexSize));
+ properties.put("Headers Etc Bytes", Long.toString(miscSize));
+ // Now output the properties table.
+ int maxKeyLength = 0;
+ Set<Map.Entry<String,String>> entrySet = properties.entrySet();
+ for (Iterator<Map.Entry<String,String>> it = entrySet.iterator(); it.hasNext();) {
+ Map.Entry<String,String> e = it.next();
+ if (e.getKey().length() > maxKeyLength) {
+ maxKeyLength = e.getKey().length();
+ }
+ }
+ for (Iterator<Map.Entry<String,String>> it = entrySet.iterator(); it.hasNext();) {
+ Map.Entry<String,String> e = it.next();
+ out.printf("%s : %s%n", Align.format(e.getKey(), maxKeyLength, Align.LEFT), e.getValue());
+ }
+ out.println();
+ reader.checkTFileDataIndex();
+ if (blockCnt > 0) {
+ String blkID = "Data-Block";
+ int blkIDWidth = Align.calculateWidth(blkID, blockCnt);
+ int blkIDWidth2 = Align.calculateWidth("", blockCnt);
+ String offset = "Offset";
+ int offsetWidth = Align.calculateWidth(offset, length);
+ String blkLen = "Length";
+ int blkLenWidth = Align.calculateWidth(blkLen, dataSize / blockCnt * 10);
+ String rawSize = "Raw-Size";
+ int rawSizeWidth = Align.calculateWidth(rawSize, dataSizeUncompressed / blockCnt * 10);
+ String records = "Records";
+ int recordsWidth = Align.calculateWidth(records, reader.getEntryCount() / blockCnt * 10);
+ String endKey = "End-Key";
+ int endKeyWidth = Math.max(endKey.length(), maxKeySampleLen * 2 + 5);
+
+ out.printf("%s %s %s %s %s %s%n", Align.format(blkID, blkIDWidth, Align.CENTER), Align.format(offset, offsetWidth, Align.CENTER),
+ Align.format(blkLen, blkLenWidth, Align.CENTER), Align.format(rawSize, rawSizeWidth, Align.CENTER),
+ Align.format(records, recordsWidth, Align.CENTER), Align.format(endKey, endKeyWidth, Align.LEFT));
+
+ for (int i = 0; i < blockCnt; ++i) {
+ BlockRegion region = reader.readerBCF.dataIndex.getBlockRegionList().get(i);
+ TFileIndexEntry indexEntry = reader.tfileIndex.getEntry(i);
+ out.printf("%s %s %s %s %s ", Align.format(Align.format(i, blkIDWidth2, Align.ZERO_PADDED), blkIDWidth, Align.LEFT),
+ Align.format(region.getOffset(), offsetWidth, Align.LEFT), Align.format(region.getCompressedSize(), blkLenWidth, Align.LEFT),
+ Align.format(region.getRawSize(), rawSizeWidth, Align.LEFT), Align.format(indexEntry.kvEntries, recordsWidth, Align.LEFT));
+ byte[] key = indexEntry.key;
+ boolean asAscii = true;
+ int sampleLen = Math.min(maxKeySampleLen, key.length);
+ for (int j = 0; j < sampleLen; ++j) {
+ byte b = key[j];
+ if ((b < 32 && b != 9) || (b == 127)) {
+ asAscii = false;
+ }
+ }
+ if (!asAscii) {
+ out.print("0X");
+ for (int j = 0; j < sampleLen; ++j) {
+ byte b = key[i];
+ out.printf("%X", b);
+ }
+ } else {
+ out.print(new String(key, 0, sampleLen, Constants.UTF8));
+ }
+ if (sampleLen < key.length) {
+ out.print("...");
+ }
+ out.println();
+ }
+ }
+
+ out.println();
+ if (metaBlkCnt > 0) {
+ String name = "Meta-Block";
+ int maxNameLen = 0;
+ Set<Map.Entry<String,MetaIndexEntry>> metaBlkEntrySet = reader.readerBCF.metaIndex.index.entrySet();
+ for (Iterator<Map.Entry<String,MetaIndexEntry>> it = metaBlkEntrySet.iterator(); it.hasNext();) {
+ Map.Entry<String,MetaIndexEntry> e = it.next();
+ if (e.getKey().length() > maxNameLen) {
+ maxNameLen = e.getKey().length();
+ }
+ }
+ int nameWidth = Math.max(name.length(), maxNameLen);
+ String offset = "Offset";
+ int offsetWidth = Align.calculateWidth(offset, length);
+ String blkLen = "Length";
+ int blkLenWidth = Align.calculateWidth(blkLen, metaSize / metaBlkCnt * 10);
+ String rawSize = "Raw-Size";
+ int rawSizeWidth = Align.calculateWidth(rawSize, metaSizeUncompressed / metaBlkCnt * 10);
+ String compression = "Compression";
+ int compressionWidth = compression.length();
+ out.printf("%s %s %s %s %s%n", Align.format(name, nameWidth, Align.CENTER), Align.format(offset, offsetWidth, Align.CENTER),
+ Align.format(blkLen, blkLenWidth, Align.CENTER), Align.format(rawSize, rawSizeWidth, Align.CENTER),
+ Align.format(compression, compressionWidth, Align.LEFT));
+
+ for (Iterator<Map.Entry<String,MetaIndexEntry>> it = metaBlkEntrySet.iterator(); it.hasNext();) {
+ Map.Entry<String,MetaIndexEntry> e = it.next();
+ String blkName = e.getValue().getMetaName();
+ BlockRegion region = e.getValue().getRegion();
+ String blkCompression = e.getValue().getCompressionAlgorithm().getName();
+ out.printf("%s %s %s %s %s%n", Align.format(blkName, nameWidth, Align.LEFT), Align.format(region.getOffset(), offsetWidth, Align.LEFT),
+ Align.format(region.getCompressedSize(), blkLenWidth, Align.LEFT), Align.format(region.getRawSize(), rawSizeWidth, Align.LEFT),
+ Align.format(blkCompression, compressionWidth, Align.LEFT));
+ }
+ }
+ } finally {
+ IOUtils.cleanup(LOG, reader, fsdis);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Utils.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Utils.java
index 45a59f6,0000000..9131d30
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Utils.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Utils.java
@@@ -1,485 -1,0 +1,474 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.accumulo.core.file.rfile.bcfile;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.hadoop.io.Text;
+
+/**
+ * Supporting Utility classes used by TFile, and shared by users of TFile.
+ */
+public final class Utils {
+
+ /**
+ * Prevent the instantiation of Utils.
+ */
+ private Utils() {
+ // nothing
+ }
+
+ /**
+ * Encoding an integer into a variable-length encoding format. Synonymous to <code>Utils#writeVLong(out, n)</code>.
+ *
+ * @param out
+ * output stream
+ * @param n
+ * The integer to be encoded
- * @throws IOException
+ * @see Utils#writeVLong(DataOutput, long)
+ */
+ public static void writeVInt(DataOutput out, int n) throws IOException {
+ writeVLong(out, n);
+ }
+
+ /**
+ * Encoding a Long integer into a variable-length encoding format.
+ * <ul>
+ * <li>if n in [-32, 127): encode in one byte with the actual value. Otherwise,
+ * <li>if n in [-20*2^8, 20*2^8): encode in two bytes: byte[0] = n/256 - 52; byte[1]=n&0xff. Otherwise,
+ * <li>if n IN [-16*2^16, 16*2^16): encode in three bytes: byte[0]=n/2^16 - 88; byte[1]=(n>>8)&0xff; byte[2]=n&0xff. Otherwise,
+ * <li>if n in [-8*2^24, 8*2^24): encode in four bytes: byte[0]=n/2^24 - 112; byte[1] = (n>>16)&0xff; byte[2] = (n>>8)&0xff; byte[3]=n&0xff. Otherwise:
+ * <li>if n in [-2^31, 2^31): encode in five bytes: byte[0]=-125; byte[1] = (n>>24)&0xff; byte[2]=(n>>16)&0xff; byte[3]=(n>>8)&0xff; byte[4]=n&0xff;
+ * <li>if n in [-2^39, 2^39): encode in six bytes: byte[0]=-124; byte[1] = (n>>32)&0xff; byte[2]=(n>>24)&0xff; byte[3]=(n>>16)&0xff; byte[4]=(n>>8)&0xff;
+ * byte[5]=n&0xff
+ * <li>if n in [-2^47, 2^47): encode in seven bytes: byte[0]=-123; byte[1] = (n>>40)&0xff; byte[2]=(n>>32)&0xff; byte[3]=(n>>24)&0xff; byte[4]=(n>>16)&0xff;
+ * byte[5]=(n>>8)&0xff; byte[6]=n&0xff;
+ * <li>if n in [-2^55, 2^55): encode in eight bytes: byte[0]=-122; byte[1] = (n>>48)&0xff; byte[2] = (n>>40)&0xff; byte[3]=(n>>32)&0xff; byte[4]=(n>>24)&0xff;
+ * byte[5]=(n>>16)&0xff; byte[6]=(n>>8)&0xff; byte[7]=n&0xff;
+ * <li>if n in [-2^63, 2^63): encode in nine bytes: byte[0]=-121; byte[1] = (n>>54)&0xff; byte[2] = (n>>48)&0xff; byte[3] = (n>>40)&0xff;
+ * byte[4]=(n>>32)&0xff; byte[5]=(n>>24)&0xff; byte[6]=(n>>16)&0xff; byte[7]=(n>>8)&0xff; byte[8]=n&0xff;
+ * </ul>
+ *
+ * @param out
+ * output stream
+ * @param n
+ * the integer number
- * @throws IOException
+ */
+ @SuppressWarnings("fallthrough")
+ public static void writeVLong(DataOutput out, long n) throws IOException {
+ if ((n < 128) && (n >= -32)) {
+ out.writeByte((int) n);
+ return;
+ }
+
+ long un = (n < 0) ? ~n : n;
+ // how many bytes do we need to represent the number with sign bit?
+ int len = (Long.SIZE - Long.numberOfLeadingZeros(un)) / 8 + 1;
+ int firstByte = (int) (n >> ((len - 1) * 8));
+ switch (len) {
+ case 1:
+ // fall it through to firstByte==-1, len=2.
+ firstByte >>= 8;
+ case 2:
+ if ((firstByte < 20) && (firstByte >= -20)) {
+ out.writeByte(firstByte - 52);
+ out.writeByte((int) n);
+ return;
+ }
+ // fall it through to firstByte==0/-1, len=3.
+ firstByte >>= 8;
+ case 3:
+ if ((firstByte < 16) && (firstByte >= -16)) {
+ out.writeByte(firstByte - 88);
+ out.writeShort((int) n);
+ return;
+ }
+ // fall it through to firstByte==0/-1, len=4.
+ firstByte >>= 8;
+ case 4:
+ if ((firstByte < 8) && (firstByte >= -8)) {
+ out.writeByte(firstByte - 112);
+ out.writeShort(((int) n) >>> 8);
+ out.writeByte((int) n);
+ return;
+ }
+ out.writeByte(len - 129);
+ out.writeInt((int) n);
+ return;
+ case 5:
+ out.writeByte(len - 129);
+ out.writeInt((int) (n >>> 8));
+ out.writeByte((int) n);
+ return;
+ case 6:
+ out.writeByte(len - 129);
+ out.writeInt((int) (n >>> 16));
+ out.writeShort((int) n);
+ return;
+ case 7:
+ out.writeByte(len - 129);
+ out.writeInt((int) (n >>> 24));
+ out.writeShort((int) (n >>> 8));
+ out.writeByte((int) n);
+ return;
+ case 8:
+ out.writeByte(len - 129);
+ out.writeLong(n);
+ return;
+ default:
+ throw new RuntimeException("Internel error");
+ }
+ }
+
+ /**
+ * Decoding the variable-length integer. Synonymous to <code>(int)Utils#readVLong(in)</code>.
+ *
+ * @param in
+ * input stream
+ * @return the decoded integer
- * @throws IOException
+ *
+ * @see Utils#readVLong(DataInput)
+ */
+ public static int readVInt(DataInput in) throws IOException {
+ long ret = readVLong(in);
+ if ((ret > Integer.MAX_VALUE) || (ret < Integer.MIN_VALUE)) {
+ throw new RuntimeException("Number too large to be represented as Integer");
+ }
+ return (int) ret;
+ }
+
+ /**
+ * Decoding the variable-length integer. Suppose the value of the first byte is FB, and the following bytes are NB[*].
+ * <ul>
+ * <li>if (FB >= -32), return (long)FB;
+ * <li>if (FB in [-72, -33]), return (FB+52)<<8 + NB[0]&0xff;
+ * <li>if (FB in [-104, -73]), return (FB+88)<<16 + (NB[0]&0xff)<<8 + NB[1]&0xff;
+ * <li>if (FB in [-120, -105]), return (FB+112)<<24 + (NB[0]&0xff)<<16 + (NB[1]&0xff)<<8 + NB[2]&0xff;
+ * <li>if (FB in [-128, -121]), return interpret NB[FB+129] as a signed big-endian integer.
+ *
+ * @param in
+ * input stream
+ * @return the decoded long integer.
- * @throws IOException
+ */
+
+ public static long readVLong(DataInput in) throws IOException {
+ int firstByte = in.readByte();
+ if (firstByte >= -32) {
+ return firstByte;
+ }
+
+ switch ((firstByte + 128) / 8) {
+ case 11:
+ case 10:
+ case 9:
+ case 8:
+ case 7:
+ return ((firstByte + 52) << 8) | in.readUnsignedByte();
+ case 6:
+ case 5:
+ case 4:
+ case 3:
+ return ((firstByte + 88) << 16) | in.readUnsignedShort();
+ case 2:
+ case 1:
+ return ((firstByte + 112) << 24) | (in.readUnsignedShort() << 8) | in.readUnsignedByte();
+ case 0:
+ int len = firstByte + 129;
+ switch (len) {
+ case 4:
+ return in.readInt();
+ case 5:
+ return ((long) in.readInt()) << 8 | in.readUnsignedByte();
+ case 6:
+ return ((long) in.readInt()) << 16 | in.readUnsignedShort();
+ case 7:
+ return ((long) in.readInt()) << 24 | (in.readUnsignedShort() << 8) | in.readUnsignedByte();
+ case 8:
+ return in.readLong();
+ default:
+ throw new IOException("Corrupted VLong encoding");
+ }
+ default:
+ throw new RuntimeException("Internal error");
+ }
+ }
+
+ /**
+ * Write a String as a VInt n, followed by n Bytes as in Text format.
- *
- * @param out
- * @param s
- * @throws IOException
+ */
+ public static void writeString(DataOutput out, String s) throws IOException {
+ if (s != null) {
+ Text text = new Text(s);
+ byte[] buffer = text.getBytes();
+ int len = text.getLength();
+ writeVInt(out, len);
+ out.write(buffer, 0, len);
+ } else {
+ writeVInt(out, -1);
+ }
+ }
+
+ /**
+ * Read a String as a VInt n, followed by n Bytes in Text format.
+ *
+ * @param in
+ * The input stream.
+ * @return The string
- * @throws IOException
+ */
+ public static String readString(DataInput in) throws IOException {
+ int length = readVInt(in);
+ if (length == -1)
+ return null;
+ byte[] buffer = new byte[length];
+ in.readFully(buffer);
+ return Text.decode(buffer);
+ }
+
+ /**
+ * A generic Version class. We suggest applications built on top of TFile use this class to maintain version information in their meta blocks.
+ *
+ * A version number consists of a major version and a minor version. The suggested usage of major and minor version number is to increment major version
+ * number when the new storage format is not backward compatible, and increment the minor version otherwise.
+ */
+ public static final class Version implements Comparable<Version> {
+ private final short major;
+ private final short minor;
+
+ /**
+ * Construct the Version object by reading from the input stream.
+ *
+ * @param in
+ * input stream
- * @throws IOException
+ */
+ public Version(DataInput in) throws IOException {
+ major = in.readShort();
+ minor = in.readShort();
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param major
+ * major version.
+ * @param minor
+ * minor version.
+ */
+ public Version(short major, short minor) {
+ this.major = major;
+ this.minor = minor;
+ }
+
+ /**
+ * Write the object to a DataOutput. The serialized format of the Version is major version followed by minor version, both as big-endian short integers.
+ *
+ * @param out
+ * The DataOutput object.
- * @throws IOException
+ */
+ public void write(DataOutput out) throws IOException {
+ out.writeShort(major);
+ out.writeShort(minor);
+ }
+
+ /**
+ * Get the major version.
+ *
+ * @return Major version.
+ */
+ public int getMajor() {
+ return major;
+ }
+
+ /**
+ * Get the minor version.
+ *
+ * @return The minor version.
+ */
+ public int getMinor() {
+ return minor;
+ }
+
+ /**
+ * Get the size of the serialized Version object.
+ *
+ * @return serialized size of the version object.
+ */
+ public static int size() {
+ return (Short.SIZE + Short.SIZE) / Byte.SIZE;
+ }
+
+ /**
+ * Return a string representation of the version.
+ */
+ @Override
+ public String toString() {
+ return new StringBuilder("v").append(major).append(".").append(minor).toString();
+ }
+
+ /**
+ * Test compatibility.
+ *
+ * @param other
+ * The Version object to test compatibility with.
+ * @return true if both versions have the same major version number; false otherwise.
+ */
+ public boolean compatibleWith(Version other) {
+ return major == other.major;
+ }
+
+ /**
+ * Compare this version with another version.
+ */
+ @Override
+ public int compareTo(Version that) {
+ if (major != that.major) {
+ return major - that.major;
+ }
+ return minor - that.minor;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other)
+ return true;
+ if (!(other instanceof Version))
+ return false;
+ return compareTo((Version) other) == 0;
+ }
+
+ @Override
+ public int hashCode() {
+ return (major << 16 + minor);
+ }
+ }
+
+ /**
+ * Lower bound binary search. Find the index to the first element in the list that compares greater than or equal to key.
+ *
+ * @param <T>
+ * Type of the input key.
+ * @param list
+ * The list
+ * @param key
+ * The input key.
+ * @param cmp
+ * Comparator for the key.
+ * @return The index to the desired element if it exists; or list.size() otherwise.
+ */
+ public static <T> int lowerBound(List<? extends T> list, T key, Comparator<? super T> cmp) {
+ int low = 0;
+ int high = list.size();
+
+ while (low < high) {
+ int mid = (low + high) >>> 1;
+ T midVal = list.get(mid);
+ int ret = cmp.compare(midVal, key);
+ if (ret < 0)
+ low = mid + 1;
+ else
+ high = mid;
+ }
+ return low;
+ }
+
+ /**
+ * Upper bound binary search. Find the index to the first element in the list that compares greater than the input key.
+ *
+ * @param <T>
+ * Type of the input key.
+ * @param list
+ * The list
+ * @param key
+ * The input key.
+ * @param cmp
+ * Comparator for the key.
+ * @return The index to the desired element if it exists; or list.size() otherwise.
+ */
+ public static <T> int upperBound(List<? extends T> list, T key, Comparator<? super T> cmp) {
+ int low = 0;
+ int high = list.size();
+
+ while (low < high) {
+ int mid = (low + high) >>> 1;
+ T midVal = list.get(mid);
+ int ret = cmp.compare(midVal, key);
+ if (ret <= 0)
+ low = mid + 1;
+ else
+ high = mid;
+ }
+ return low;
+ }
+
+ /**
+ * Lower bound binary search. Find the index to the first element in the list that compares greater than or equal to key.
+ *
+ * @param <T>
+ * Type of the input key.
+ * @param list
+ * The list
+ * @param key
+ * The input key.
+ * @return The index to the desired element if it exists; or list.size() otherwise.
+ */
+ public static <T> int lowerBound(List<? extends Comparable<? super T>> list, T key) {
+ int low = 0;
+ int high = list.size();
+
+ while (low < high) {
+ int mid = (low + high) >>> 1;
+ Comparable<? super T> midVal = list.get(mid);
+ int ret = midVal.compareTo(key);
+ if (ret < 0)
+ low = mid + 1;
+ else
+ high = mid;
+ }
+ return low;
+ }
+
+ /**
+ * Upper bound binary search. Find the index to the first element in the list that compares greater than the input key.
+ *
+ * @param <T>
+ * Type of the input key.
+ * @param list
+ * The list
+ * @param key
+ * The input key.
+ * @return The index to the desired element if it exists; or list.size() otherwise.
+ */
+ public static <T> int upperBound(List<? extends Comparable<? super T>> list, T key) {
+ int low = 0;
+ int high = list.size();
+
+ while (low < high) {
+ int mid = (low + high) >>> 1;
+ Comparable<? super T> midVal = list.get(mid);
+ int ret = midVal.compareTo(key);
+ if (ret <= 0)
+ low = mid + 1;
+ else
+ high = mid;
+ }
+ return low;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
index 54a1333,0000000..5b7b05c
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
@@@ -1,249 -1,0 +1,243 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+
+/**
+ * A Combiner that decodes each Value to type V before reducing, then encodes the result of typedReduce back to Value.
+ *
+ * Subclasses must implement a typedReduce method: public V typedReduce(Key key, Iterator<V> iter);
+ *
+ * This typedReduce method will be passed the most recent Key and an iterator over the Values (translated to Vs) for all non-deleted versions of that Key.
+ *
+ * Subclasses may implement a switch on the "type" variable to choose an Encoder in their init method.
+ */
+public abstract class TypedValueCombiner<V> extends Combiner {
+ private Encoder<V> encoder = null;
+ private boolean lossy = false;
+
+ protected static final String LOSSY = "lossy";
+
+ /**
+ * A Java Iterator that translates an Iterator<Value> to an Iterator<V> using the decode method of an Encoder.
+ */
+ private static class VIterator<V> implements Iterator<V> {
+ private Iterator<Value> source;
+ private Encoder<V> encoder;
+ private boolean lossy;
+
+ /**
+ * Constructs an Iterator<V> from an Iterator<Value>
+ *
+ * @param iter
+ * The source iterator
+ *
+ * @param encoder
+ * The Encoder whose decode method is used to translate from Value to V
+ *
+ * @param lossy
+ * Determines whether to error on failure to decode or ignore and move on
+ */
+ VIterator(Iterator<Value> iter, Encoder<V> encoder, boolean lossy) {
+ this.source = iter;
+ this.encoder = encoder;
+ this.lossy = lossy;
+ }
+
+ V next = null;
+ boolean hasNext = false;
+
+ @Override
+ public boolean hasNext() {
+ if (hasNext)
+ return true;
+
+ while (true) {
+ if (!source.hasNext())
+ return false;
+ try {
+ next = encoder.decode(source.next().get());
+ return hasNext = true;
+ } catch (ValueFormatException vfe) {
+ if (!lossy)
+ throw vfe;
+ }
+ }
+ }
+
+ @Override
+ public V next() {
+ if (!hasNext && !hasNext())
+ throw new NoSuchElementException();
+ V toRet = next;
+ next = null;
+ hasNext = false;
+ return toRet;
+ }
+
+ @Override
+ public void remove() {
+ source.remove();
+ }
+ }
+
+ /**
+ * An interface for translating from byte[] to V and back.
+ */
+ public static interface Encoder<V> {
+ public byte[] encode(V v);
+
+ public V decode(byte[] b) throws ValueFormatException;
+ }
+
+ /**
+ * Sets the Encoder<V> used to translate Values to V and back.
- *
- * @param encoder
+ */
+ protected void setEncoder(Encoder<V> encoder) {
+ this.encoder = encoder;
+ }
+
+ /**
+ * Instantiates and sets the Encoder<V> used to translate Values to V and back.
+ *
- * @param encoderClass
+ * @throws IllegalArgumentException
+ * if ClassNotFoundException, InstantiationException, or IllegalAccessException occurs
+ */
+ protected void setEncoder(String encoderClass) {
+ try {
+ @SuppressWarnings("unchecked")
+ Class<? extends Encoder<V>> clazz = (Class<? extends Encoder<V>>) AccumuloVFSClassLoader.loadClass(encoderClass, Encoder.class);
+ encoder = clazz.newInstance();
+ } catch (ClassNotFoundException e) {
+ throw new IllegalArgumentException(e);
+ } catch (InstantiationException e) {
+ throw new IllegalArgumentException(e);
+ } catch (IllegalAccessException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+
+ /**
+ * Tests whether v remains the same when encoded and decoded with the current encoder.
+ *
- * @param v
+ * @throws IllegalStateException
+ * if an encoder has not been set.
+ * @throws IllegalArgumentException
+ * if the test fails.
+ */
+ protected void testEncoder(V v) {
+ if (encoder == null)
+ throw new IllegalStateException("encoder has not been initialized");
+ testEncoder(encoder, v);
+ }
+
+ /**
+ * Tests whether v remains the same when encoded and decoded with the given encoder.
+ *
- * @param encoder
- * @param v
+ * @throws IllegalArgumentException
+ * if the test fails.
+ */
+ public static <V> void testEncoder(Encoder<V> encoder, V v) {
+ try {
+ if (!v.equals(encoder.decode(encoder.encode(v))))
+ throw new IllegalArgumentException("something wrong with " + encoder.getClass().getName() + " -- doesn't encode and decode " + v + " properly");
+ } catch (ClassCastException e) {
+ throw new IllegalArgumentException(encoder.getClass().getName() + " doesn't encode " + v.getClass().getName());
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ TypedValueCombiner<V> newInstance = (TypedValueCombiner<V>) super.deepCopy(env);
+ newInstance.setEncoder(encoder);
+ return newInstance;
+ }
+
+ @Override
+ public Value reduce(Key key, Iterator<Value> iter) {
+ return new Value(encoder.encode(typedReduce(key, new VIterator<V>(iter, encoder, lossy))));
+ }
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ super.init(source, options, env);
+ setLossyness(options);
+ }
+
+ private void setLossyness(Map<String,String> options) {
+ String loss = options.get(LOSSY);
+ if (loss == null)
+ lossy = false;
+ else
+ lossy = Boolean.parseBoolean(loss);
+ }
+
+ @Override
+ public IteratorOptions describeOptions() {
+ IteratorOptions io = super.describeOptions();
+ io.addNamedOption(LOSSY, "if true, failed decodes are ignored. Otherwise combiner will error on failed decodes (default false): <TRUE|FALSE>");
+ return io;
+ }
+
+ @Override
+ public boolean validateOptions(Map<String,String> options) {
+ if (super.validateOptions(options) == false)
+ return false;
+ try {
+ setLossyness(options);
+ } catch (Exception e) {
+ throw new IllegalArgumentException("bad boolean " + LOSSY + ":" + options.get(LOSSY));
+ }
+ return true;
+ }
+
+ /**
+ * A convenience method to set the "lossy" option on a TypedValueCombiner. If true, the combiner will ignore any values which fail to decode. Otherwise, the
+ * combiner will throw an error which will interrupt the action (and prevent potential data loss). False is the default behavior.
+ *
+ * @param is
+ * iterator settings object to configure
+ * @param lossy
+ * if true the combiner will ignored values which fail to decode; otherwise error.
+ */
+ public static void setLossyness(IteratorSetting is, boolean lossy) {
+ is.addOption(LOSSY, Boolean.toString(lossy));
+ }
+
+ /**
+ * Reduces a list of V into a single V.
+ *
+ * @param key
+ * The most recent version of the Key being reduced.
+ *
+ * @param iter
+ * An iterator over the V for different versions of the key.
+ *
+ * @return The combined V.
+ */
+ public abstract V typedReduce(Key key, Iterator<V> iter);
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/ValueFormatException.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/ValueFormatException.java
index 7bb2228,0000000..7ede7fe
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/ValueFormatException.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/ValueFormatException.java
@@@ -1,40 -1,0 +1,34 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators;
+
+/**
+ * Exception used for TypedValueCombiner and it's Encoders decode() function
+ */
+public class ValueFormatException extends IllegalArgumentException {
+
- /**
- * @param string
- */
+ public ValueFormatException(String string) {
+ super(string);
+ }
+
- /**
- * @param nfe
- */
+ public ValueFormatException(Exception nfe) {
+ super(nfe);
+ }
+
+ private static final long serialVersionUID = 4170291568272971821L;
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/system/MapFileIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/system/MapFileIterator.java
index e5fe62a,0000000..37a234c
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/system/MapFileIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/system/MapFileIterator.java
@@@ -1,162 -1,0 +1,156 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators.system;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileSKVIterator;
+import org.apache.accumulo.core.file.NoSuchMetaStoreException;
+import org.apache.accumulo.core.file.map.MapFileUtil;
+import org.apache.accumulo.core.iterators.IterationInterruptedException;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.MapFile.Reader;
+import org.apache.log4j.Logger;
+
+public class MapFileIterator implements FileSKVIterator {
+ private static final Logger log = Logger.getLogger(MapFileIterator.class);
+
+ private Reader reader;
+ private Value topValue;
+ private Key topKey;
+ private AtomicBoolean interruptFlag;
+ private int interruptCheckCount = 0;
+ private FileSystem fs;
+ private String dirName;
+
- /**
- * @param acuconf
- * @param fs
- * @param dir
- * @param conf
- * @throws IOException
- */
+ public MapFileIterator(AccumuloConfiguration acuconf, FileSystem fs, String dir, Configuration conf) throws IOException {
+ this.reader = MapFileUtil.openMapFile(acuconf, fs, dir, conf);
+ this.fs = fs;
+ this.dirName = dir;
+ }
+
+ @Override
+ public void setInterruptFlag(AtomicBoolean flag) {
+ this.interruptFlag = flag;
+ }
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean hasTop() {
+ return topKey != null;
+ }
+
+ @Override
+ public void next() throws IOException {
+ if (interruptFlag != null && interruptCheckCount++ % 100 == 0 && interruptFlag.get())
+ throw new IterationInterruptedException();
+
+ reader.next(topKey, topValue);
+ }
+
++ @Override
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
+ if (columnFamilies.size() != 0 || inclusive) {
+ throw new IllegalArgumentException("I do not know how to filter column families");
+ }
+
+ if (range == null)
+ throw new IllegalArgumentException("Cannot seek to null range");
+
+ if (interruptFlag != null && interruptFlag.get())
+ throw new IterationInterruptedException();
+
+ Key key = range.getStartKey();
+ if (key == null) {
+ key = new Key();
+ }
+
+ reader.seek(key);
+
+ while (hasTop() && range.beforeStartKey(getTopKey())) {
+ next();
+ }
+ }
+
+ @Override
+ public Key getTopKey() {
+ return topKey;
+ }
+
+ @Override
+ public Value getTopValue() {
+ return topValue;
+ }
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ try {
+ SortedKeyValueIterator<Key,Value> other = env.reserveMapFileReader(dirName);
+ ((InterruptibleIterator) other).setInterruptFlag(interruptFlag);
+ log.debug("deep copying MapFile: " + this + " -> " + other);
+ return other;
+ } catch (IOException e) {
+ log.error("failed to clone map file reader", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public Key getFirstKey() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Key getLastKey() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DataInputStream getMetaStore(String name) throws IOException {
+ Path path = new Path(this.dirName, name);
+ if (!fs.exists(path))
+ throw new NoSuchMetaStoreException("name = " + name);
+ return fs.open(path);
+ }
+
+ @Override
+ public void closeDeepCopies() throws IOException {
+ // nothing to do, deep copies are externally managed/closed
+ }
+
+ @Override
+ public void close() throws IOException {
+ reader.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
index 4f8207c,0000000..86798dd
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
@@@ -1,104 -1,0 +1,101 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators.user;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Map;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.Filter;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+
+/**
+ * This iterator provides exact string matching. It searches both the Key and Value for the string. The string to match is specified by the "term" option.
+ */
+public class GrepIterator extends Filter {
+
+ private byte term[];
+
+ @Override
+ public boolean accept(Key k, Value v) {
+ return match(v.get()) || match(k.getRowData()) || match(k.getColumnFamilyData()) || match(k.getColumnQualifierData());
+ }
+
+ private boolean match(ByteSequence bs) {
+ return indexOf(bs.getBackingArray(), bs.offset(), bs.length(), term) >= 0;
+ }
+
+ private boolean match(byte[] ba) {
+ return indexOf(ba, 0, ba.length, term) >= 0;
+ }
+
+ // copied code below from java string and modified
+
+ private static int indexOf(byte[] source, int sourceOffset, int sourceCount, byte[] target) {
+ byte first = target[0];
+ int targetCount = target.length;
+ int max = sourceOffset + (sourceCount - targetCount);
+
+ for (int i = sourceOffset; i <= max; i++) {
+ /* Look for first character. */
+ if (source[i] != first) {
+ while (++i <= max && source[i] != first)
+ continue;
+ }
+
+ /* Found first character, now look at the rest of v2 */
+ if (i <= max) {
+ int j = i + 1;
+ int end = j + targetCount - 1;
+ for (int k = 1; j < end && source[j] == target[k]; j++, k++)
+ continue;
+
+ if (j == end) {
+ /* Found whole string. */
+ return i - sourceOffset;
+ }
+ }
+ }
+ return -1;
+ }
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ GrepIterator copy = (GrepIterator) super.deepCopy(env);
+ copy.term = Arrays.copyOf(term, term.length);
+ return copy;
+ }
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ super.init(source, options, env);
+ term = options.get("term").getBytes(Constants.UTF8);
+ }
+
+ /**
+ * Encode the grep term as an option for a ScanIterator
- *
- * @param cfg
- * @param term
+ */
+ public static void setTerm(IteratorSetting cfg, String term) {
+ cfg.addOption("term", term);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
index 447200b,0000000..39cba6d
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
@@@ -1,558 -1,0 +1,548 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators.user;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+/**
+ * This iterator facilitates document-partitioned indexing. It involves grouping a set of documents together and indexing those documents into a single row of
+ * an Accumulo table. This allows a tablet server to perform boolean AND operations on terms in the index.
+ *
+ * The table structure should have the following form:
+ *
+ * row: shardID, colfam: term, colqual: docID
+ *
+ * When you configure this iterator with a set of terms (column families), it will return only the docIDs that appear with all of the specified terms. The
+ * result will have an empty column family, as follows:
+ *
+ * row: shardID, colfam: (empty), colqual: docID
+ *
+ * This iterator is commonly used with BatchScanner or AccumuloInputFormat, to parallelize the search over all shardIDs.
+ *
+ * This iterator will *ignore* any columnFamilies passed to {@link #seek(Range, Collection, boolean)} as it performs intersections over terms. Extending classes
+ * should override the {@link TermSource#seekColfams} in their implementation's {@link #init(SortedKeyValueIterator, Map, IteratorEnvironment)} method.
+ *
+ * README.shard in docs/examples shows an example of using the IntersectingIterator.
+ */
+public class IntersectingIterator implements SortedKeyValueIterator<Key,Value> {
+
+ protected Text nullText = new Text();
+
+ protected Text getPartition(Key key) {
+ return key.getRow();
+ }
+
+ protected Text getTerm(Key key) {
+ return key.getColumnFamily();
+ }
+
+ protected Text getDocID(Key key) {
+ return key.getColumnQualifier();
+ }
+
+ protected Key buildKey(Text partition, Text term) {
+ return new Key(partition, (term == null) ? nullText : term);
+ }
+
+ protected Key buildKey(Text partition, Text term, Text docID) {
+ return new Key(partition, (term == null) ? nullText : term, docID);
+ }
+
+ protected Key buildFollowingPartitionKey(Key key) {
+ return key.followingKey(PartialKey.ROW);
+ }
+
+ protected static final Logger log = Logger.getLogger(IntersectingIterator.class);
+
+ public static class TermSource {
+ public SortedKeyValueIterator<Key,Value> iter;
+ public Text term;
+ public Collection<ByteSequence> seekColfams;
+ public boolean notFlag;
+
+ public TermSource(TermSource other) {
+ this.iter = other.iter;
+ this.term = other.term;
+ this.notFlag = other.notFlag;
+ this.seekColfams = other.seekColfams;
+ }
+
+ public TermSource(SortedKeyValueIterator<Key,Value> iter, Text term) {
+ this(iter, term, false);
+ }
+
+ public TermSource(SortedKeyValueIterator<Key,Value> iter, Text term, boolean notFlag) {
+ this.iter = iter;
+ this.term = term;
+ this.notFlag = notFlag;
+ // The desired column families for this source is the term itself
+ this.seekColfams = Collections.<ByteSequence> singletonList(new ArrayByteSequence(term.getBytes(), 0, term.getLength()));
+ }
+
+ public String getTermString() {
+ return (this.term == null) ? "Iterator" : this.term.toString();
+ }
+ }
+
+ protected TermSource[] sources;
+ int sourcesCount = 0;
+
+ Range overallRange;
+
+ // query-time settings
+ protected Text currentPartition = null;
+ protected Text currentDocID = new Text(emptyByteArray);
+ static final byte[] emptyByteArray = new byte[0];
+
+ protected Key topKey = null;
+ protected Value value = new Value(emptyByteArray);
+
+ public IntersectingIterator() {}
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ return new IntersectingIterator(this, env);
+ }
+
+ private IntersectingIterator(IntersectingIterator other, IteratorEnvironment env) {
+ if (other.sources != null) {
+ sourcesCount = other.sourcesCount;
+ sources = new TermSource[sourcesCount];
+ for (int i = 0; i < sourcesCount; i++) {
+ sources[i] = new TermSource(other.sources[i].iter.deepCopy(env), other.sources[i].term);
+ }
+ }
+ }
+
+ @Override
+ public Key getTopKey() {
+ return topKey;
+ }
+
+ @Override
+ public Value getTopValue() {
+ // we don't really care about values
+ return value;
+ }
+
+ @Override
+ public boolean hasTop() {
+ return currentPartition != null;
+ }
+
+ // precondition: currentRow is not null
+ private boolean seekOneSource(int sourceID) throws IOException {
+ // find the next key in the appropriate column family that is at or beyond the cursor (currentRow, currentCQ)
+ // advance the cursor if this source goes beyond it
+ // return whether we advanced the cursor
+
+ // within this loop progress must be made in one of the following forms:
+ // - currentRow or currentCQ must be increased
+ // - the given source must advance its iterator
+ // this loop will end when any of the following criteria are met
+ // - the iterator for the given source is pointing to the key (currentRow, columnFamilies[sourceID], currentCQ)
+ // - the given source is out of data and currentRow is set to null
+ // - the given source has advanced beyond the endRow and currentRow is set to null
+ boolean advancedCursor = false;
+
+ if (sources[sourceID].notFlag) {
+ while (true) {
+ if (sources[sourceID].iter.hasTop() == false) {
+ // an empty column that you are negating is a valid condition
+ break;
+ }
+ // check if we're past the end key
+ int endCompare = -1;
+ // we should compare the row to the end of the range
+ if (overallRange.getEndKey() != null) {
+ endCompare = overallRange.getEndKey().getRow().compareTo(sources[sourceID].iter.getTopKey().getRow());
+ if ((!overallRange.isEndKeyInclusive() && endCompare <= 0) || endCompare < 0) {
+ // an empty column that you are negating is a valid condition
+ break;
+ }
+ }
+ int partitionCompare = currentPartition.compareTo(getPartition(sources[sourceID].iter.getTopKey()));
+ // check if this source is already at or beyond currentRow
+ // if not, then seek to at least the current row
+
+ if (partitionCompare > 0) {
+ // seek to at least the currentRow
+ Key seekKey = buildKey(currentPartition, sources[sourceID].term);
+ sources[sourceID].iter.seek(new Range(seekKey, true, null, false), sources[sourceID].seekColfams, true);
+ continue;
+ }
+ // check if this source has gone beyond currentRow
+ // if so, this is a valid condition for negation
+ if (partitionCompare < 0) {
+ break;
+ }
+ // we have verified that the current source is positioned in currentRow
+ // now we must make sure we're in the right columnFamily in the current row
+ // Note: Iterators are auto-magically set to the correct columnFamily
+ if (sources[sourceID].term != null) {
+ int termCompare = sources[sourceID].term.compareTo(getTerm(sources[sourceID].iter.getTopKey()));
+ // check if this source is already on the right columnFamily
+ // if not, then seek forwards to the right columnFamily
+ if (termCompare > 0) {
+ Key seekKey = buildKey(currentPartition, sources[sourceID].term, currentDocID);
+ sources[sourceID].iter.seek(new Range(seekKey, true, null, false), sources[sourceID].seekColfams, true);
+ continue;
+ }
+ // check if this source is beyond the right columnFamily
+ // if so, then this is a valid condition for negating
+ if (termCompare < 0) {
+ break;
+ }
+ }
+
+ // we have verified that we are in currentRow and the correct column family
+ // make sure we are at or beyond columnQualifier
+ Text docID = getDocID(sources[sourceID].iter.getTopKey());
+ int docIDCompare = currentDocID.compareTo(docID);
+ // If we are past the target, this is a valid result
+ if (docIDCompare < 0) {
+ break;
+ }
+ // if this source is not yet at the currentCQ then advance in this source
+ if (docIDCompare > 0) {
+ // seek forwards
+ Key seekKey = buildKey(currentPartition, sources[sourceID].term, currentDocID);
+ sources[sourceID].iter.seek(new Range(seekKey, true, null, false), sources[sourceID].seekColfams, true);
+ continue;
+ }
+ // if we are equal to the target, this is an invalid result.
+ // Force the entire process to go to the next row.
+ // We are advancing column 0 because we forced that column to not contain a !
+ // when we did the init()
+ if (docIDCompare == 0) {
+ sources[0].iter.next();
+ advancedCursor = true;
+ break;
+ }
+ }
+ } else {
+ while (true) {
+ if (sources[sourceID].iter.hasTop() == false) {
+ currentPartition = null;
+ // setting currentRow to null counts as advancing the cursor
+ return true;
+ }
+ // check if we're past the end key
+ int endCompare = -1;
+ // we should compare the row to the end of the range
+
+ if (overallRange.getEndKey() != null) {
+ endCompare = overallRange.getEndKey().getRow().compareTo(sources[sourceID].iter.getTopKey().getRow());
+ if ((!overallRange.isEndKeyInclusive() && endCompare <= 0) || endCompare < 0) {
+ currentPartition = null;
+ // setting currentRow to null counts as advancing the cursor
+ return true;
+ }
+ }
+ int partitionCompare = currentPartition.compareTo(getPartition(sources[sourceID].iter.getTopKey()));
+ // check if this source is already at or beyond currentRow
+ // if not, then seek to at least the current row
+ if (partitionCompare > 0) {
+ // seek to at least the currentRow
+ Key seekKey = buildKey(currentPartition, sources[sourceID].term);
+ sources[sourceID].iter.seek(new Range(seekKey, true, null, false), sources[sourceID].seekColfams, true);
+ continue;
+ }
+ // check if this source has gone beyond currentRow
+ // if so, advance currentRow
+ if (partitionCompare < 0) {
+ currentPartition.set(getPartition(sources[sourceID].iter.getTopKey()));
+ currentDocID.set(emptyByteArray);
+ advancedCursor = true;
+ continue;
+ }
+ // we have verified that the current source is positioned in currentRow
+ // now we must make sure we're in the right columnFamily in the current row
+ // Note: Iterators are auto-magically set to the correct columnFamily
+
+ if (sources[sourceID].term != null) {
+ int termCompare = sources[sourceID].term.compareTo(getTerm(sources[sourceID].iter.getTopKey()));
+ // check if this source is already on the right columnFamily
+ // if not, then seek forwards to the right columnFamily
+ if (termCompare > 0) {
+ Key seekKey = buildKey(currentPartition, sources[sourceID].term, currentDocID);
+ sources[sourceID].iter.seek(new Range(seekKey, true, null, false), sources[sourceID].seekColfams, true);
+ continue;
+ }
+ // check if this source is beyond the right columnFamily
+ // if so, then seek to the next row
+ if (termCompare < 0) {
+ // we're out of entries in the current row, so seek to the next one
+ // byte[] currentRowBytes = currentRow.getBytes();
+ // byte[] nextRow = new byte[currentRowBytes.length + 1];
+ // System.arraycopy(currentRowBytes, 0, nextRow, 0, currentRowBytes.length);
+ // nextRow[currentRowBytes.length] = (byte)0;
+ // // we should reuse text objects here
+ // sources[sourceID].seek(new Key(new Text(nextRow),columnFamilies[sourceID]));
+ if (endCompare == 0) {
+ // we're done
+ currentPartition = null;
+ // setting currentRow to null counts as advancing the cursor
+ return true;
+ }
+ Key seekKey = buildFollowingPartitionKey(sources[sourceID].iter.getTopKey());
+ sources[sourceID].iter.seek(new Range(seekKey, true, null, false), sources[sourceID].seekColfams, true);
+ continue;
+ }
+ }
+ // we have verified that we are in currentRow and the correct column family
+ // make sure we are at or beyond columnQualifier
+ Text docID = getDocID(sources[sourceID].iter.getTopKey());
+ int docIDCompare = currentDocID.compareTo(docID);
+ // if this source has advanced beyond the current column qualifier then advance currentCQ and return true
+ if (docIDCompare < 0) {
+ currentDocID.set(docID);
+ advancedCursor = true;
+ break;
+ }
+ // if this source is not yet at the currentCQ then seek in this source
+ if (docIDCompare > 0) {
+ // seek forwards
+ Key seekKey = buildKey(currentPartition, sources[sourceID].term, currentDocID);
+ sources[sourceID].iter.seek(new Range(seekKey, true, null, false), sources[sourceID].seekColfams, true);
+ continue;
+ }
+ // this source is at the current row, in its column family, and at currentCQ
+ break;
+ }
+ }
+ return advancedCursor;
+ }
+
+ @Override
+ public void next() throws IOException {
+ if (currentPartition == null) {
+ return;
+ }
+ // precondition: the current row is set up and the sources all have the same column qualifier
+ // while we don't have a match, seek in the source with the smallest column qualifier
+ sources[0].iter.next();
+ advanceToIntersection();
+ }
+
+ protected void advanceToIntersection() throws IOException {
+ boolean cursorChanged = true;
+ while (cursorChanged) {
+ // seek all of the sources to at least the highest seen column qualifier in the current row
+ cursorChanged = false;
+ for (int i = 0; i < sourcesCount; i++) {
+ if (currentPartition == null) {
+ topKey = null;
+ return;
+ }
+ if (seekOneSource(i)) {
+ cursorChanged = true;
+ break;
+ }
+ }
+ }
+ topKey = buildKey(currentPartition, nullText, currentDocID);
+ }
+
+ public static String stringTopKey(SortedKeyValueIterator<Key,Value> iter) {
+ if (iter.hasTop())
+ return iter.getTopKey().toString();
+ return "";
+ }
+
+ private static final String columnFamiliesOptionName = "columnFamilies";
+ private static final String notFlagOptionName = "notFlag";
+
+ /**
- * @param columns
+ * @return encoded columns
+ */
+ protected static String encodeColumns(Text[] columns) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < columns.length; i++) {
+ sb.append(new String(Base64.encodeBase64(TextUtil.getBytes(columns[i])), Constants.UTF8));
+ sb.append('\n');
+ }
+ return sb.toString();
+ }
+
+ /**
- * @param flags
+ * @return encoded flags
+ */
+ protected static String encodeBooleans(boolean[] flags) {
+ byte[] bytes = new byte[flags.length];
+ for (int i = 0; i < flags.length; i++) {
+ if (flags[i])
+ bytes[i] = 1;
+ else
+ bytes[i] = 0;
+ }
+ return new String(Base64.encodeBase64(bytes), Constants.UTF8);
+ }
+
+ protected static Text[] decodeColumns(String columns) {
+ String[] columnStrings = columns.split("\n");
+ Text[] columnTexts = new Text[columnStrings.length];
+ for (int i = 0; i < columnStrings.length; i++) {
+ columnTexts[i] = new Text(Base64.decodeBase64(columnStrings[i].getBytes(Constants.UTF8)));
+ }
+ return columnTexts;
+ }
+
+ /**
- * @param flags
+ * @return decoded flags
+ */
+ protected static boolean[] decodeBooleans(String flags) {
+ // return null of there were no flags
+ if (flags == null)
+ return null;
+
+ byte[] bytes = Base64.decodeBase64(flags.getBytes(Constants.UTF8));
+ boolean[] bFlags = new boolean[bytes.length];
+ for (int i = 0; i < bytes.length; i++) {
+ if (bytes[i] == 1)
+ bFlags[i] = true;
+ else
+ bFlags[i] = false;
+ }
+ return bFlags;
+ }
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ Text[] terms = decodeColumns(options.get(columnFamiliesOptionName));
+ boolean[] notFlag = decodeBooleans(options.get(notFlagOptionName));
+
+ if (terms.length < 2) {
+ throw new IllegalArgumentException("IntersectionIterator requires two or more columns families");
+ }
+
+ // Scan the not flags.
+ // There must be at least one term that isn't negated
+ // And we are going to re-order such that the first term is not a ! term
+ if (notFlag == null) {
+ notFlag = new boolean[terms.length];
+ for (int i = 0; i < terms.length; i++)
+ notFlag[i] = false;
+ }
+ if (notFlag[0]) {
+ for (int i = 1; i < notFlag.length; i++) {
+ if (notFlag[i] == false) {
+ Text swapFamily = new Text(terms[0]);
+ terms[0].set(terms[i]);
+ terms[i].set(swapFamily);
+ notFlag[0] = false;
+ notFlag[i] = true;
+ break;
+ }
+ }
+ if (notFlag[0]) {
+ throw new IllegalArgumentException("IntersectionIterator requires at lest one column family without not");
+ }
+ }
+
+ sources = new TermSource[terms.length];
+ sources[0] = new TermSource(source, terms[0]);
+ for (int i = 1; i < terms.length; i++) {
+ sources[i] = new TermSource(source.deepCopy(env), terms[i], notFlag[i]);
+ }
+ sourcesCount = terms.length;
+ }
+
+ @Override
+ public void seek(Range range, Collection<ByteSequence> seekColumnFamilies, boolean inclusive) throws IOException {
+ overallRange = new Range(range);
+ currentPartition = new Text();
+ currentDocID.set(emptyByteArray);
+
+ // seek each of the sources to the right column family within the row given by key
+ for (int i = 0; i < sourcesCount; i++) {
+ Key sourceKey;
+ if (range.getStartKey() != null) {
+ if (range.getStartKey().getColumnQualifier() != null) {
+ sourceKey = buildKey(getPartition(range.getStartKey()), sources[i].term, range.getStartKey().getColumnQualifier());
+ } else {
+ sourceKey = buildKey(getPartition(range.getStartKey()), sources[i].term);
+ }
+ // Seek only to the term for this source as a column family
+ sources[i].iter.seek(new Range(sourceKey, true, null, false), sources[i].seekColfams, true);
+ } else {
+ // Seek only to the term for this source as a column family
+ sources[i].iter.seek(range, sources[i].seekColfams, true);
+ }
+ }
+ advanceToIntersection();
+ }
+
+ public void addSource(SortedKeyValueIterator<Key,Value> source, IteratorEnvironment env, Text term, boolean notFlag) {
+ // Check if we have space for the added Source
+ if (sources == null) {
+ sources = new TermSource[1];
+ } else {
+ // allocate space for node, and copy current tree.
+ // TODO: Should we change this to an ArrayList so that we can just add() ? - ACCUMULO-1309
+ TermSource[] localSources = new TermSource[sources.length + 1];
+ int currSource = 0;
+ for (TermSource myTerm : sources) {
+ // TODO: Do I need to call new here? or can I just re-use the term? - ACCUMULO-1309
+ localSources[currSource] = new TermSource(myTerm);
+ currSource++;
+ }
+ sources = localSources;
+ }
+ sources[sourcesCount] = new TermSource(source.deepCopy(env), term, notFlag);
+ sourcesCount++;
+ }
+
+ /**
+ * Encode the columns to be used when iterating.
- *
- * @param cfg
- * @param columns
+ */
+ public static void setColumnFamilies(IteratorSetting cfg, Text[] columns) {
+ if (columns.length < 2)
+ throw new IllegalArgumentException("Must supply at least two terms to intersect");
+ cfg.addOption(IntersectingIterator.columnFamiliesOptionName, IntersectingIterator.encodeColumns(columns));
+ }
+
+ /**
+ * Encode columns and NOT flags indicating which columns should be negated (docIDs will be excluded if matching negated columns, instead of included).
- *
- * @param cfg
- * @param columns
- * @param notFlags
+ */
+ public static void setColumnFamilies(IteratorSetting cfg, Text[] columns, boolean[] notFlags) {
+ if (columns.length < 2)
+ throw new IllegalArgumentException("Must supply at least two terms to intersect");
+ if (columns.length != notFlags.length)
+ throw new IllegalArgumentException("columns and notFlags arrays must be the same length");
+ setColumnFamilies(cfg, columns);
+ cfg.addOption(IntersectingIterator.notFlagOptionName, IntersectingIterator.encodeBooleans(notFlags));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/user/RowFilter.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/RowFilter.java
index a232796,0000000..2d2fa74
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/RowFilter.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/RowFilter.java
@@@ -1,165 -1,0 +1,164 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators.user;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.WrappingIterator;
+import org.apache.hadoop.io.Text;
+
+/**
+ * This iterator makes it easy to select rows that meet a given criteria. Its an alternative to the {@link WholeRowIterator}. There are a few things to consider
+ * when deciding which one to use.
+ *
+ * First the WholeRowIterator requires that the row fit in memory and that the entire row is read before a decision is made. This iterator has neither
+ * requirement, it allows seeking within a row to avoid reading the entire row to make a decision. So even if your rows fit into memory, this extending this
+ * iterator may be better choice because you can seek.
+ *
+ * Second the WholeRowIterator is currently the only way to achieve row isolation with the {@link BatchScanner}. With the normal {@link Scanner} row isolation
+ * can be enabled and this Iterator may be used.
+ *
+ * Third the row acceptance test will be executed every time this Iterator is seeked. If the row is large, then the row will fetched in batches of key/values.
+ * As each batch is fetched the test may be re-executed because the iterator stack is reseeked for each batch. The batch size may be increased to reduce the
+ * number of times the test is executed. With the normal Scanner, if isolation is enabled then it will read an entire row w/o seeking this iterator.
+ *
+ */
+public abstract class RowFilter extends WrappingIterator {
+
+ private RowIterator decisionIterator;
+ private Collection<ByteSequence> columnFamilies;
+ Text currentRow;
+ private boolean inclusive;
+ private Range range;
+ private boolean hasTop;
+
+ private static class RowIterator extends WrappingIterator {
+ private Range rowRange;
+ private boolean hasTop;
+
+ RowIterator(SortedKeyValueIterator<Key,Value> source) {
+ super.setSource(source);
+ }
+
+ void setRow(Range row) {
+ this.rowRange = row;
+ }
+
+ @Override
+ public boolean hasTop() {
+ return hasTop && super.hasTop();
+ }
+
+ @Override
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
+
+ range = rowRange.clip(range, true);
+ if (range == null) {
+ hasTop = false;
+ } else {
+ hasTop = true;
+ super.seek(range, columnFamilies, inclusive);
+ }
+ }
+ }
+
+ private void skipRows() throws IOException {
+ SortedKeyValueIterator<Key,Value> source = getSource();
+ while (source.hasTop()) {
+ Text row = source.getTopKey().getRow();
+
+ if (currentRow != null && currentRow.equals(row))
+ break;
+
+ Range rowRange = new Range(row);
+ decisionIterator.setRow(rowRange);
+ decisionIterator.seek(rowRange, columnFamilies, inclusive);
+
+ if (acceptRow(decisionIterator)) {
+ currentRow = row;
+ break;
+ } else {
+ currentRow = null;
+ int count = 0;
+ while (source.hasTop() && count < 10 && source.getTopKey().getRow().equals(row)) {
+ count++;
+ source.next();
+ }
+
+ if (source.hasTop() && source.getTopKey().getRow().equals(row)) {
+ Range nextRow = new Range(row, false, null, false);
+ nextRow = range.clip(nextRow, true);
+ if (nextRow == null)
+ hasTop = false;
+ else
+ source.seek(nextRow, columnFamilies, inclusive);
+ }
+ }
+ }
+ }
+
+ /**
+ * Implementation should return false to suppress a row.
+ *
+ *
+ * @param rowIterator
+ * - An iterator over the row. This iterator is confined to the row. Seeking past the end of the row will return no data. Seeking before the row will
+ * always set top to the first column in the current row. By default this iterator will only see the columns the parent was seeked with. To see more
+ * columns reseek this iterator with those columns.
+ * @return false if a row should be suppressed, otherwise true.
- * @throws IOException
+ */
+ public abstract boolean acceptRow(SortedKeyValueIterator<Key,Value> rowIterator) throws IOException;
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ super.init(source, options, env);
+ this.decisionIterator = new RowIterator(source.deepCopy(env));
+ }
+
+ @Override
+ public boolean hasTop() {
+ return hasTop && super.hasTop();
+ }
+
+ @Override
+ public void next() throws IOException {
+ super.next();
+ skipRows();
+ }
+
+ @Override
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
+ super.seek(range, columnFamilies, inclusive);
+ this.columnFamilies = columnFamilies;
+ this.inclusive = inclusive;
+ this.range = range;
+ currentRow = null;
+ hasTop = true;
+ skipRows();
+
+ }
+}
[59/64] [abbrv] Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 6d73125,0000000..e2510d7
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@@ -1,3913 -1,0 +1,3913 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver;
+
+import static org.apache.accumulo.server.problems.ProblemType.TABLET_LOAD;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.net.Socket;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TimerTask;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.RunnableFuture;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.management.ObjectName;
+import javax.management.StandardMBean;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.CompressedIterators;
+import org.apache.accumulo.core.client.impl.CompressedIterators.IterConfig;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.TabletType;
+import org.apache.accumulo.core.client.impl.Translator;
+import org.apache.accumulo.core.client.impl.Translator.TKeyExtentTranslator;
+import org.apache.accumulo.core.client.impl.Translator.TRangeTranslator;
++import org.apache.accumulo.core.client.impl.Translators;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
- import org.apache.accumulo.core.client.impl.Translators;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.constraints.Constraint.Environment;
+import org.apache.accumulo.core.constraints.Violations;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Column;
+import org.apache.accumulo.core.data.ConstraintViolationSummary;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.thrift.InitialMultiScan;
+import org.apache.accumulo.core.data.thrift.InitialScan;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.data.thrift.MapFileInfo;
+import org.apache.accumulo.core.data.thrift.MultiScanResult;
+import org.apache.accumulo.core.data.thrift.ScanResult;
+import org.apache.accumulo.core.data.thrift.TCMResult;
+import org.apache.accumulo.core.data.thrift.TCMStatus;
+import org.apache.accumulo.core.data.thrift.TColumn;
+import org.apache.accumulo.core.data.thrift.TCondition;
+import org.apache.accumulo.core.data.thrift.TConditionalMutation;
+import org.apache.accumulo.core.data.thrift.TConditionalSession;
+import org.apache.accumulo.core.data.thrift.TKey;
+import org.apache.accumulo.core.data.thrift.TKeyExtent;
+import org.apache.accumulo.core.data.thrift.TKeyValue;
+import org.apache.accumulo.core.data.thrift.TMutation;
+import org.apache.accumulo.core.data.thrift.TRange;
+import org.apache.accumulo.core.data.thrift.UpdateErrors;
+import org.apache.accumulo.core.iterators.IterationInterruptedException;
+import org.apache.accumulo.core.master.thrift.Compacting;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.master.thrift.TableInfo;
+import org.apache.accumulo.core.master.thrift.TabletLoadState;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.security.AuthorizationContainer;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.tabletserver.log.LogEntry;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
+import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
+import org.apache.accumulo.core.tabletserver.thrift.NoSuchScanIDException;
+import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
+import org.apache.accumulo.core.tabletserver.thrift.ScanState;
+import org.apache.accumulo.core.tabletserver.thrift.ScanType;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Processor;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.LoggingRunnable;
+import org.apache.accumulo.core.util.MapCounter;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.ServerServices;
+import org.apache.accumulo.core.util.ServerServices.Service;
+import org.apache.accumulo.core.util.SimpleThreadPool;
+import org.apache.accumulo.core.util.Stat;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockWatcher;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.ServerOpts;
+import org.apache.accumulo.server.client.ClientServiceHandler;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.conf.TableConfiguration;
+import org.apache.accumulo.server.data.ServerMutation;
+import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManager.FileType;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.fs.VolumeUtil;
+import org.apache.accumulo.server.master.recovery.RecoveryPath;
+import org.apache.accumulo.server.master.state.Assignment;
+import org.apache.accumulo.server.master.state.DistributedStoreException;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletLocationState;
+import org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException;
+import org.apache.accumulo.server.master.state.TabletStateStore;
+import org.apache.accumulo.server.master.state.ZooTabletStateStore;
+import org.apache.accumulo.server.metrics.AbstractMetricsImpl;
+import org.apache.accumulo.server.problems.ProblemReport;
+import org.apache.accumulo.server.problems.ProblemReports;
+import org.apache.accumulo.server.security.AuditedSecurityOperation;
+import org.apache.accumulo.server.security.SecurityOperation;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.util.FileSystemMonitor;
+import org.apache.accumulo.server.util.Halt;
+import org.apache.accumulo.server.util.MasterMetadataUtil;
+import org.apache.accumulo.server.util.MetadataTableUtil;
+import org.apache.accumulo.server.util.TServerUtils;
+import org.apache.accumulo.server.util.TServerUtils.ServerAddress;
+import org.apache.accumulo.server.util.time.RelativeTime;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
+import org.apache.accumulo.server.zookeeper.TransactionWatcher;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.accumulo.start.classloader.vfs.ContextManager;
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.instrument.Trace;
+import org.apache.accumulo.trace.instrument.thrift.TraceWrap;
+import org.apache.accumulo.trace.thrift.TInfo;
+import org.apache.accumulo.tserver.Compactor.CompactionInfo;
+import org.apache.accumulo.tserver.RowLocks.RowLock;
+import org.apache.accumulo.tserver.Tablet.CommitSession;
+import org.apache.accumulo.tserver.Tablet.KVEntry;
+import org.apache.accumulo.tserver.Tablet.LookupResult;
+import org.apache.accumulo.tserver.Tablet.MinorCompactionReason;
+import org.apache.accumulo.tserver.Tablet.ScanBatch;
+import org.apache.accumulo.tserver.Tablet.Scanner;
+import org.apache.accumulo.tserver.Tablet.SplitInfo;
+import org.apache.accumulo.tserver.Tablet.TConstraintViolationException;
+import org.apache.accumulo.tserver.Tablet.TabletClosedException;
+import org.apache.accumulo.tserver.TabletServerResourceManager.TabletResourceManager;
+import org.apache.accumulo.tserver.TabletStatsKeeper.Operation;
+import org.apache.accumulo.tserver.compaction.MajorCompactionReason;
+import org.apache.accumulo.tserver.data.ServerConditionalMutation;
+import org.apache.accumulo.tserver.log.DfsLogger;
+import org.apache.accumulo.tserver.log.LogSorter;
+import org.apache.accumulo.tserver.log.MutationReceiver;
+import org.apache.accumulo.tserver.log.TabletServerLogger;
+import org.apache.accumulo.tserver.mastermessage.MasterMessage;
+import org.apache.accumulo.tserver.mastermessage.SplitReportMessage;
+import org.apache.accumulo.tserver.mastermessage.TabletStatusMessage;
+import org.apache.accumulo.tserver.metrics.TabletServerMBean;
+import org.apache.accumulo.tserver.metrics.TabletServerMinCMetrics;
+import org.apache.accumulo.tserver.metrics.TabletServerScanMetrics;
+import org.apache.accumulo.tserver.metrics.TabletServerUpdateMetrics;
+import org.apache.commons.collections.map.LRUMap;
+import org.apache.hadoop.fs.FSError;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+import org.apache.thrift.TProcessor;
+import org.apache.thrift.TServiceClient;
+import org.apache.thrift.server.TServer;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+
+import com.google.common.net.HostAndPort;
+
+enum ScanRunState {
+ QUEUED, RUNNING, FINISHED
+}
+
+public class TabletServer extends AbstractMetricsImpl implements org.apache.accumulo.tserver.metrics.TabletServerMBean {
+ private static final Logger log = Logger.getLogger(TabletServer.class);
+
+ private static HashMap<String,Long> prevGcTime = new HashMap<String,Long>();
+ private static long lastMemorySize = 0;
+ private static long gcTimeIncreasedCount;
+
+ private static final long MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS = 1000;
+ private static final long RECENTLY_SPLIT_MILLIES = 60 * 1000;
+
+ private TabletServerLogger logger;
+
+ protected TabletServerMinCMetrics mincMetrics = new TabletServerMinCMetrics();
+
+ private ServerConfiguration serverConfig;
+ private LogSorter logSorter = null;
+
+ public TabletServer(ServerConfiguration conf, VolumeManager fs) {
+ super();
+ this.serverConfig = conf;
+ this.instance = conf.getInstance();
+ this.fs = fs;
+ this.logSorter = new LogSorter(instance, fs, getSystemConfiguration());
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ synchronized (onlineTablets) {
+ long now = System.currentTimeMillis();
+ for (Tablet tablet : onlineTablets.values())
+ try {
+ tablet.updateRates(now);
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ }
+ }
+ }, 5000, 5000);
+ }
+
+ private synchronized static void logGCInfo(AccumuloConfiguration conf) {
+ List<GarbageCollectorMXBean> gcmBeans = ManagementFactory.getGarbageCollectorMXBeans();
+ Runtime rt = Runtime.getRuntime();
+
+ StringBuilder sb = new StringBuilder("gc");
+
+ boolean sawChange = false;
+
+ long maxIncreaseInCollectionTime = 0;
+
+ for (GarbageCollectorMXBean gcBean : gcmBeans) {
+ Long prevTime = prevGcTime.get(gcBean.getName());
+ long pt = 0;
+ if (prevTime != null) {
+ pt = prevTime;
+ }
+
+ long time = gcBean.getCollectionTime();
+
+ if (time - pt != 0) {
+ sawChange = true;
+ }
+
+ long increaseInCollectionTime = time - pt;
+ sb.append(String.format(" %s=%,.2f(+%,.2f) secs", gcBean.getName(), time / 1000.0, increaseInCollectionTime / 1000.0));
+ maxIncreaseInCollectionTime = Math.max(increaseInCollectionTime, maxIncreaseInCollectionTime);
+ prevGcTime.put(gcBean.getName(), time);
+ }
+
+ long mem = rt.freeMemory();
+ if (maxIncreaseInCollectionTime == 0) {
+ gcTimeIncreasedCount = 0;
+ } else {
+ gcTimeIncreasedCount++;
+ if (gcTimeIncreasedCount > 3 && mem < rt.maxMemory() * 0.05) {
+ log.warn("Running low on memory");
+ gcTimeIncreasedCount = 0;
+ }
+ }
+
+ if (mem > lastMemorySize) {
+ sawChange = true;
+ }
+
+ String sign = "+";
+ if (mem - lastMemorySize <= 0) {
+ sign = "";
+ }
+
+ sb.append(String.format(" freemem=%,d(%s%,d) totalmem=%,d", mem, sign, (mem - lastMemorySize), rt.totalMemory()));
+
+ if (sawChange) {
+ log.debug(sb.toString());
+ }
+
+ final long keepAliveTimeout = conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT);
+ if (maxIncreaseInCollectionTime > keepAliveTimeout) {
+ Halt.halt("Garbage collection may be interfering with lock keep-alive. Halting.", -1);
+ }
+
+ lastMemorySize = mem;
+ }
+
+ private TabletStatsKeeper statsKeeper;
+
+ private static class Session {
+ long lastAccessTime;
+ long startTime;
+ String user;
+ String client = TServerUtils.clientAddress.get();
+ public boolean reserved;
+
+ public void cleanup() {}
+ }
+
+ private static class SessionManager {
+
+ SecureRandom random;
+ Map<Long,Session> sessions;
+ long maxIdle;
+
+ SessionManager(AccumuloConfiguration conf) {
+ random = new SecureRandom();
+ sessions = new HashMap<Long,Session>();
+
+ maxIdle = conf.getTimeInMillis(Property.TSERV_SESSION_MAXIDLE);
+
+ Runnable r = new Runnable() {
+ @Override
+ public void run() {
+ sweep(maxIdle);
+ }
+ };
+
+ SimpleTimer.getInstance().schedule(r, 0, Math.max(maxIdle / 2, 1000));
+ }
+
+ synchronized long createSession(Session session, boolean reserve) {
+ long sid = random.nextLong();
+
+ while (sessions.containsKey(sid)) {
+ sid = random.nextLong();
+ }
+
+ sessions.put(sid, session);
+
+ session.reserved = reserve;
+
+ session.startTime = session.lastAccessTime = System.currentTimeMillis();
+
+ return sid;
+ }
+
+ long getMaxIdleTime() {
+ return maxIdle;
+ }
+
+ /**
+ * while a session is reserved, it cannot be canceled or removed
- *
++ *
+ * @param sessionId
+ */
+
+ synchronized Session reserveSession(long sessionId) {
+ Session session = sessions.get(sessionId);
+ if (session != null) {
+ if (session.reserved)
+ throw new IllegalStateException();
+ session.reserved = true;
+ }
+
+ return session;
+
+ }
+
+ synchronized Session reserveSession(long sessionId, boolean wait) {
+ Session session = sessions.get(sessionId);
+ if (session != null) {
+ while (wait && session.reserved) {
+ try {
+ wait(1000);
+ } catch (InterruptedException e) {
+ throw new RuntimeException();
+ }
+ }
+
+ if (session.reserved)
+ throw new IllegalStateException();
+ session.reserved = true;
+ }
+
+ return session;
+
+ }
+
+ synchronized void unreserveSession(Session session) {
+ if (!session.reserved)
+ throw new IllegalStateException();
+ notifyAll();
+ session.reserved = false;
+ session.lastAccessTime = System.currentTimeMillis();
+ }
+
+ synchronized void unreserveSession(long sessionId) {
+ Session session = getSession(sessionId);
+ if (session != null)
+ unreserveSession(session);
+ }
+
+ synchronized Session getSession(long sessionId) {
+ Session session = sessions.get(sessionId);
+ if (session != null)
+ session.lastAccessTime = System.currentTimeMillis();
+ return session;
+ }
+
+ Session removeSession(long sessionId) {
+ return removeSession(sessionId, false);
+ }
+
+ Session removeSession(long sessionId, boolean unreserve) {
+ Session session = null;
+ synchronized (this) {
+ session = sessions.remove(sessionId);
+ if (unreserve && session != null)
+ unreserveSession(session);
+ }
+
+ // do clean up out side of lock..
+ if (session != null)
+ session.cleanup();
+
+ return session;
+ }
+
+ private void sweep(long maxIdle) {
+ ArrayList<Session> sessionsToCleanup = new ArrayList<Session>();
+ synchronized (this) {
+ Iterator<Session> iter = sessions.values().iterator();
+ while (iter.hasNext()) {
+ Session session = iter.next();
+ long idleTime = System.currentTimeMillis() - session.lastAccessTime;
+ if (idleTime > maxIdle && !session.reserved) {
+ iter.remove();
+ sessionsToCleanup.add(session);
+ }
+ }
+ }
+
+ // do clean up outside of lock
+ for (Session session : sessionsToCleanup) {
+ session.cleanup();
+ }
+ }
+
+ synchronized void removeIfNotAccessed(final long sessionId, long delay) {
+ Session session = sessions.get(sessionId);
+ if (session != null) {
+ final long removeTime = session.lastAccessTime;
+ TimerTask r = new TimerTask() {
+ @Override
+ public void run() {
+ Session sessionToCleanup = null;
+ synchronized (SessionManager.this) {
+ Session session2 = sessions.get(sessionId);
+ if (session2 != null && session2.lastAccessTime == removeTime && !session2.reserved) {
+ sessions.remove(sessionId);
+ sessionToCleanup = session2;
+ }
+ }
+
+ // call clean up outside of lock
+ if (sessionToCleanup != null)
+ sessionToCleanup.cleanup();
+ }
+ };
+
+ SimpleTimer.getInstance().schedule(r, delay);
+ }
+ }
+
+ public synchronized Map<String,MapCounter<ScanRunState>> getActiveScansPerTable() {
+ Map<String,MapCounter<ScanRunState>> counts = new HashMap<String,MapCounter<ScanRunState>>();
+ for (Entry<Long,Session> entry : sessions.entrySet()) {
+
+ Session session = entry.getValue();
+ @SuppressWarnings("rawtypes")
+ ScanTask nbt = null;
+ String tableID = null;
+
+ if (session instanceof ScanSession) {
+ ScanSession ss = (ScanSession) session;
+ nbt = ss.nextBatchTask;
+ tableID = ss.extent.getTableId().toString();
+ } else if (session instanceof MultiScanSession) {
+ MultiScanSession mss = (MultiScanSession) session;
+ nbt = mss.lookupTask;
+ tableID = mss.threadPoolExtent.getTableId().toString();
+ }
+
+ if (nbt == null)
+ continue;
+
+ ScanRunState srs = nbt.getScanRunState();
+
+ if (srs == ScanRunState.FINISHED)
+ continue;
+
+ MapCounter<ScanRunState> stateCounts = counts.get(tableID);
+ if (stateCounts == null) {
+ stateCounts = new MapCounter<ScanRunState>();
+ counts.put(tableID, stateCounts);
+ }
+
+ stateCounts.increment(srs, 1);
+ }
+
+ return counts;
+ }
+
+ public synchronized List<ActiveScan> getActiveScans() {
+
+ ArrayList<ActiveScan> activeScans = new ArrayList<ActiveScan>();
+
+ long ct = System.currentTimeMillis();
+
+ for (Entry<Long,Session> entry : sessions.entrySet()) {
+ Session session = entry.getValue();
+ if (session instanceof ScanSession) {
+ ScanSession ss = (ScanSession) session;
+
+ ScanState state = ScanState.RUNNING;
+
+ ScanTask<ScanBatch> nbt = ss.nextBatchTask;
+ if (nbt == null) {
+ state = ScanState.IDLE;
+ } else {
+ switch (nbt.getScanRunState()) {
+ case QUEUED:
+ state = ScanState.QUEUED;
+ break;
+ case FINISHED:
+ state = ScanState.IDLE;
+ break;
+ case RUNNING:
+ default:
+ /* do nothing */
+ break;
+ }
+ }
+
+ activeScans.add(new ActiveScan(ss.client, ss.user, ss.extent.getTableId().toString(), ct - ss.startTime, ct - ss.lastAccessTime, ScanType.SINGLE,
+ state, ss.extent.toThrift(), Translator.translate(ss.columnSet, Translators.CT), ss.ssiList, ss.ssio, ss.auths.getAuthorizationsBB()));
+
+ } else if (session instanceof MultiScanSession) {
+ MultiScanSession mss = (MultiScanSession) session;
+
+ ScanState state = ScanState.RUNNING;
+
+ ScanTask<MultiScanResult> nbt = mss.lookupTask;
+ if (nbt == null) {
+ state = ScanState.IDLE;
+ } else {
+ switch (nbt.getScanRunState()) {
+ case QUEUED:
+ state = ScanState.QUEUED;
+ break;
+ case FINISHED:
+ state = ScanState.IDLE;
+ break;
+ case RUNNING:
+ default:
+ /* do nothing */
+ break;
+ }
+ }
+
+ activeScans.add(new ActiveScan(mss.client, mss.user, mss.threadPoolExtent.getTableId().toString(), ct - mss.startTime, ct - mss.lastAccessTime,
+ ScanType.BATCH, state, mss.threadPoolExtent.toThrift(), Translator.translate(mss.columnSet, Translators.CT), mss.ssiList, mss.ssio, mss.auths
+ .getAuthorizationsBB()));
+ }
+ }
+
+ return activeScans;
+ }
+ }
+
+ static class TservConstraintEnv implements Environment {
+
+ private TCredentials credentials;
+ private SecurityOperation security;
+ private Authorizations auths;
+ private KeyExtent ke;
+
+ TservConstraintEnv(SecurityOperation secOp, TCredentials credentials) {
+ this.security = secOp;
+ this.credentials = credentials;
+ }
+
+ void setExtent(KeyExtent ke) {
+ this.ke = ke;
+ }
+
+ @Override
+ public KeyExtent getExtent() {
+ return ke;
+ }
+
+ @Override
+ public String getUser() {
+ return credentials.getPrincipal();
+ }
+
+ @Override
+ @Deprecated
+ public Authorizations getAuthorizations() {
+ if (auths == null)
+ try {
+ this.auths = security.getUserAuthorizations(credentials);
+ } catch (ThriftSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ return auths;
+ }
+
+ @Override
+ public AuthorizationContainer getAuthorizationsContainer() {
+ return new AuthorizationContainer() {
+ @Override
+ public boolean contains(ByteSequence auth) {
+ try {
+ return security.userHasAuthorizations(credentials,
+ Collections.<ByteBuffer> singletonList(ByteBuffer.wrap(auth.getBackingArray(), auth.offset(), auth.length())));
+ } catch (ThriftSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ }
+ }
+
+ private abstract class ScanTask<T> implements RunnableFuture<T> {
+
+ protected AtomicBoolean interruptFlag;
+ protected ArrayBlockingQueue<Object> resultQueue;
+ protected AtomicInteger state;
+ protected AtomicReference<ScanRunState> runState;
+
+ private static final int INITIAL = 1;
+ private static final int ADDED = 2;
+ private static final int CANCELED = 3;
+
+ ScanTask() {
+ interruptFlag = new AtomicBoolean(false);
+ runState = new AtomicReference<ScanRunState>(ScanRunState.QUEUED);
+ state = new AtomicInteger(INITIAL);
+ resultQueue = new ArrayBlockingQueue<Object>(1);
+ }
+
+ protected void addResult(Object o) {
+ if (state.compareAndSet(INITIAL, ADDED))
+ resultQueue.add(o);
+ else if (state.get() == ADDED)
+ throw new IllegalStateException("Tried to add more than one result");
+ }
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ if (!mayInterruptIfRunning)
+ throw new IllegalArgumentException("Cancel will always attempt to interupt running next batch task");
+
+ if (state.get() == CANCELED)
+ return true;
+
+ if (state.compareAndSet(INITIAL, CANCELED)) {
+ interruptFlag.set(true);
+ resultQueue = null;
+ return true;
+ }
+
+ return false;
+ }
+
+ @Override
+ public T get() throws InterruptedException, ExecutionException {
+ throw new UnsupportedOperationException();
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
+
+ ArrayBlockingQueue<Object> localRQ = resultQueue;
+
+ if (state.get() == CANCELED)
+ throw new CancellationException();
+
+ if (localRQ == null && state.get() == ADDED)
+ throw new IllegalStateException("Tried to get result twice");
+
+ Object r = localRQ.poll(timeout, unit);
+
+ // could have been canceled while waiting
+ if (state.get() == CANCELED) {
+ if (r != null)
+ throw new IllegalStateException("Nothing should have been added when in canceled state");
+
+ throw new CancellationException();
+ }
+
+ if (r == null)
+ throw new TimeoutException();
+
+ // make this method stop working now that something is being
+ // returned
+ resultQueue = null;
+
+ if (r instanceof Throwable)
+ throw new ExecutionException((Throwable) r);
+
+ return (T) r;
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return state.get() == CANCELED;
+ }
+
+ @Override
+ public boolean isDone() {
+ return runState.get().equals(ScanRunState.FINISHED);
+ }
+
+ public ScanRunState getScanRunState() {
+ return runState.get();
+ }
+
+ }
+
+ private static class ConditionalSession extends Session {
+ public TCredentials credentials;
+ public Authorizations auths;
+ public String tableId;
+ public AtomicBoolean interruptFlag;
+
+ @Override
+ public void cleanup() {
+ interruptFlag.set(true);
+ }
+ }
+
+ private static class UpdateSession extends Session {
+ public Tablet currentTablet;
+ public MapCounter<Tablet> successfulCommits = new MapCounter<Tablet>();
+ Map<KeyExtent,Long> failures = new HashMap<KeyExtent,Long>();
+ HashMap<KeyExtent,SecurityErrorCode> authFailures = new HashMap<KeyExtent,SecurityErrorCode>();
+ public Violations violations;
+ public TCredentials credentials;
+ public long totalUpdates = 0;
+ public long flushTime = 0;
+ Stat prepareTimes = new Stat();
+ Stat walogTimes = new Stat();
+ Stat commitTimes = new Stat();
+ Stat authTimes = new Stat();
+ public Map<Tablet,List<Mutation>> queuedMutations = new HashMap<Tablet,List<Mutation>>();
+ public long queuedMutationSize = 0;
+ TservConstraintEnv cenv = null;
+ }
+
+ private static class ScanSession extends Session {
+ public KeyExtent extent;
+ public HashSet<Column> columnSet;
+ public List<IterInfo> ssiList;
+ public Map<String,Map<String,String>> ssio;
+ public Authorizations auths;
+ public long entriesReturned = 0;
+ public Stat nbTimes = new Stat();
+ public long batchCount = 0;
+ public volatile ScanTask<ScanBatch> nextBatchTask;
+ public AtomicBoolean interruptFlag;
+ public Scanner scanner;
+ public long readaheadThreshold = Constants.SCANNER_DEFAULT_READAHEAD_THRESHOLD;
+
+ @Override
+ public void cleanup() {
+ try {
+ if (nextBatchTask != null)
+ nextBatchTask.cancel(true);
+ } finally {
+ if (scanner != null)
+ scanner.close();
+ }
+ }
+
+ }
+
+ private static class MultiScanSession extends Session {
+ HashSet<Column> columnSet;
+ Map<KeyExtent,List<Range>> queries;
+ public List<IterInfo> ssiList;
+ public Map<String,Map<String,String>> ssio;
+ public Authorizations auths;
+
+ // stats
+ int numRanges;
+ int numTablets;
+ int numEntries;
+ long totalLookupTime;
+
+ public volatile ScanTask<MultiScanResult> lookupTask;
+ public KeyExtent threadPoolExtent;
+
+ @Override
+ public void cleanup() {
+ if (lookupTask != null)
+ lookupTask.cancel(true);
+ }
+ }
+
+ /**
+ * This little class keeps track of writes in progress and allows readers to wait for writes that started before the read. It assumes that the operation ids
+ * are monotonically increasing.
- *
++ *
+ */
+ static class WriteTracker {
+ private static AtomicLong operationCounter = new AtomicLong(1);
+ private Map<TabletType,TreeSet<Long>> inProgressWrites = new EnumMap<TabletType,TreeSet<Long>>(TabletType.class);
+
+ WriteTracker() {
+ for (TabletType ttype : TabletType.values()) {
+ inProgressWrites.put(ttype, new TreeSet<Long>());
+ }
+ }
+
+ synchronized long startWrite(TabletType ttype) {
+ long operationId = operationCounter.getAndIncrement();
+ inProgressWrites.get(ttype).add(operationId);
+ return operationId;
+ }
+
+ synchronized void finishWrite(long operationId) {
+ if (operationId == -1)
+ return;
+
+ boolean removed = false;
+
+ for (TabletType ttype : TabletType.values()) {
+ removed = inProgressWrites.get(ttype).remove(operationId);
+ if (removed)
+ break;
+ }
+
+ if (!removed) {
+ throw new IllegalArgumentException("Attempted to finish write not in progress, operationId " + operationId);
+ }
+
+ this.notifyAll();
+ }
+
+ synchronized void waitForWrites(TabletType ttype) {
+ long operationId = operationCounter.getAndIncrement();
+ while (inProgressWrites.get(ttype).floor(operationId) != null) {
+ try {
+ this.wait();
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ }
+ }
+ }
+
+ public long startWrite(Set<Tablet> keySet) {
+ if (keySet.size() == 0)
+ return -1;
+
+ ArrayList<KeyExtent> extents = new ArrayList<KeyExtent>(keySet.size());
+
+ for (Tablet tablet : keySet)
+ extents.add(tablet.getExtent());
+
+ return startWrite(TabletType.type(extents));
+ }
+ }
+
+ public AccumuloConfiguration getSystemConfiguration() {
+ return serverConfig.getConfiguration();
+ }
+
+ TransactionWatcher watcher = new TransactionWatcher();
+
+ private class ThriftClientHandler extends ClientServiceHandler implements TabletClientService.Iface {
+
+ SessionManager sessionManager;
+
+ AccumuloConfiguration acuConf = getSystemConfiguration();
+
+ TabletServerUpdateMetrics updateMetrics = new TabletServerUpdateMetrics();
+
+ TabletServerScanMetrics scanMetrics = new TabletServerScanMetrics();
+
+ WriteTracker writeTracker = new WriteTracker();
+
+ private RowLocks rowLocks = new RowLocks();
+
+ ThriftClientHandler() {
+ super(instance, watcher, fs);
+ log.debug(ThriftClientHandler.class.getName() + " created");
+ sessionManager = new SessionManager(getSystemConfiguration());
+ // Register the metrics MBean
+ try {
+ updateMetrics.register();
+ scanMetrics.register();
+ } catch (Exception e) {
+ log.error("Exception registering MBean with MBean Server", e);
+ }
+ }
+
+ @Override
+ public List<TKeyExtent> bulkImport(TInfo tinfo, TCredentials credentials, long tid, Map<TKeyExtent,Map<String,MapFileInfo>> files, boolean setTime)
+ throws ThriftSecurityException {
+
+ if (!security.canPerformSystemActions(credentials))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ List<TKeyExtent> failures = new ArrayList<TKeyExtent>();
+
+ for (Entry<TKeyExtent,Map<String,MapFileInfo>> entry : files.entrySet()) {
+ TKeyExtent tke = entry.getKey();
+ Map<String,MapFileInfo> fileMap = entry.getValue();
+ Map<FileRef,MapFileInfo> fileRefMap = new HashMap<FileRef,MapFileInfo>();
+ for (Entry<String,MapFileInfo> mapping : fileMap.entrySet()) {
+ Path path = new Path(mapping.getKey());
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
+ path = ns.makeQualified(path);
+ fileRefMap.put(new FileRef(path.toString(), path), mapping.getValue());
+ }
+
+ Tablet importTablet = onlineTablets.get(new KeyExtent(tke));
+
+ if (importTablet == null) {
+ failures.add(tke);
+ } else {
+ try {
+ importTablet.importMapFiles(tid, fileRefMap, setTime);
+ } catch (IOException ioe) {
+ log.info("files " + fileMap.keySet() + " not imported to " + new KeyExtent(tke) + ": " + ioe.getMessage());
+ failures.add(tke);
+ }
+ }
+ }
+ return failures;
+ }
+
+ private class NextBatchTask extends ScanTask<ScanBatch> {
+
+ private long scanID;
+
+ NextBatchTask(long scanID, AtomicBoolean interruptFlag) {
+ this.scanID = scanID;
+ this.interruptFlag = interruptFlag;
+
+ if (interruptFlag.get())
+ cancel(true);
+ }
+
+ @Override
+ public void run() {
+
+ final ScanSession scanSession = (ScanSession) sessionManager.getSession(scanID);
+ String oldThreadName = Thread.currentThread().getName();
+
+ try {
+ if (isCancelled() || scanSession == null)
+ return;
+
+ runState.set(ScanRunState.RUNNING);
+
+ Thread.currentThread().setName(
+ "User: " + scanSession.user + " Start: " + scanSession.startTime + " Client: " + scanSession.client + " Tablet: " + scanSession.extent);
+
+ Tablet tablet = onlineTablets.get(scanSession.extent);
+
+ if (tablet == null) {
+ addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
+ return;
+ }
+
+ long t1 = System.currentTimeMillis();
+ ScanBatch batch = scanSession.scanner.read();
+ long t2 = System.currentTimeMillis();
+ scanSession.nbTimes.addStat(t2 - t1);
+
+ // there should only be one thing on the queue at a time, so
+ // it should be ok to call add()
+ // instead of put()... if add() fails because queue is at
+ // capacity it means there is code
+ // problem somewhere
+ addResult(batch);
+ } catch (TabletClosedException e) {
+ addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
+ } catch (IterationInterruptedException iie) {
+ if (!isCancelled()) {
+ log.warn("Iteration interrupted, when scan not cancelled", iie);
+ addResult(iie);
+ }
+ } catch (TooManyFilesException tmfe) {
+ addResult(tmfe);
+ } catch (Throwable e) {
+ log.warn("exception while scanning tablet " + (scanSession == null ? "(unknown)" : scanSession.extent), e);
+ addResult(e);
+ } finally {
+ runState.set(ScanRunState.FINISHED);
+ Thread.currentThread().setName(oldThreadName);
+ }
+
+ }
+ }
+
+ private class LookupTask extends ScanTask<MultiScanResult> {
+
+ private long scanID;
+
+ LookupTask(long scanID) {
+ this.scanID = scanID;
+ }
+
+ @Override
+ public void run() {
+ MultiScanSession session = (MultiScanSession) sessionManager.getSession(scanID);
+ String oldThreadName = Thread.currentThread().getName();
+
+ try {
+ if (isCancelled() || session == null)
+ return;
+
+ TableConfiguration acuTableConf = ServerConfiguration.getTableConfiguration(instance, session.threadPoolExtent.getTableId().toString());
+ long maxResultsSize = acuTableConf.getMemoryInBytes(Property.TABLE_SCAN_MAXMEM);
+
+ runState.set(ScanRunState.RUNNING);
+ Thread.currentThread().setName("Client: " + session.client + " User: " + session.user + " Start: " + session.startTime + " Table: ");
+
+ long bytesAdded = 0;
+ long maxScanTime = 4000;
+
+ long startTime = System.currentTimeMillis();
+
+ ArrayList<KVEntry> results = new ArrayList<KVEntry>();
+ Map<KeyExtent,List<Range>> failures = new HashMap<KeyExtent,List<Range>>();
+ ArrayList<KeyExtent> fullScans = new ArrayList<KeyExtent>();
+ KeyExtent partScan = null;
+ Key partNextKey = null;
+ boolean partNextKeyInclusive = false;
+
+ Iterator<Entry<KeyExtent,List<Range>>> iter = session.queries.entrySet().iterator();
+
+ // check the time so that the read ahead thread is not monopolized
+ while (iter.hasNext() && bytesAdded < maxResultsSize && (System.currentTimeMillis() - startTime) < maxScanTime) {
+ Entry<KeyExtent,List<Range>> entry = iter.next();
+
+ iter.remove();
+
+ // check that tablet server is serving requested tablet
+ Tablet tablet = onlineTablets.get(entry.getKey());
+ if (tablet == null) {
+ failures.put(entry.getKey(), entry.getValue());
+ continue;
+ }
+ Thread.currentThread().setName(
+ "Client: " + session.client + " User: " + session.user + " Start: " + session.startTime + " Tablet: " + entry.getKey().toString());
+
+ LookupResult lookupResult;
+ try {
+
+ // do the following check to avoid a race condition
+ // between setting false below and the task being
+ // canceled
+ if (isCancelled())
+ interruptFlag.set(true);
+
+ lookupResult = tablet.lookup(entry.getValue(), session.columnSet, session.auths, results, maxResultsSize - bytesAdded, session.ssiList,
+ session.ssio, interruptFlag);
+
+ // if the tablet was closed it it possible that the
+ // interrupt flag was set.... do not want it set for
+ // the next
+ // lookup
+ interruptFlag.set(false);
+
+ } catch (IOException e) {
+ log.warn("lookup failed for tablet " + entry.getKey(), e);
+ throw new RuntimeException(e);
+ }
+
+ bytesAdded += lookupResult.bytesAdded;
+
+ if (lookupResult.unfinishedRanges.size() > 0) {
+ if (lookupResult.closed) {
+ failures.put(entry.getKey(), lookupResult.unfinishedRanges);
+ } else {
+ session.queries.put(entry.getKey(), lookupResult.unfinishedRanges);
+ partScan = entry.getKey();
+ partNextKey = lookupResult.unfinishedRanges.get(0).getStartKey();
+ partNextKeyInclusive = lookupResult.unfinishedRanges.get(0).isStartKeyInclusive();
+ }
+ } else {
+ fullScans.add(entry.getKey());
+ }
+ }
+
+ long finishTime = System.currentTimeMillis();
+ session.totalLookupTime += (finishTime - startTime);
+ session.numEntries += results.size();
+
+ // convert everything to thrift before adding result
+ List<TKeyValue> retResults = new ArrayList<TKeyValue>();
+ for (KVEntry entry : results)
+ retResults.add(new TKeyValue(entry.key.toThrift(), ByteBuffer.wrap(entry.value)));
+ Map<TKeyExtent,List<TRange>> retFailures = Translator.translate(failures, Translators.KET, new Translator.ListTranslator<Range,TRange>(Translators.RT));
+ List<TKeyExtent> retFullScans = Translator.translate(fullScans, Translators.KET);
+ TKeyExtent retPartScan = null;
+ TKey retPartNextKey = null;
+ if (partScan != null) {
+ retPartScan = partScan.toThrift();
+ retPartNextKey = partNextKey.toThrift();
+ }
+ // add results to queue
+ addResult(new MultiScanResult(retResults, retFailures, retFullScans, retPartScan, retPartNextKey, partNextKeyInclusive, session.queries.size() != 0));
+ } catch (IterationInterruptedException iie) {
+ if (!isCancelled()) {
+ log.warn("Iteration interrupted, when scan not cancelled", iie);
+ addResult(iie);
+ }
+ } catch (Throwable e) {
+ log.warn("exception while doing multi-scan ", e);
+ addResult(e);
+ } finally {
+ Thread.currentThread().setName(oldThreadName);
+ runState.set(ScanRunState.FINISHED);
+ }
+ }
+ }
+
+ @Override
+ public InitialScan startScan(TInfo tinfo, TCredentials credentials, TKeyExtent textent, TRange range, List<TColumn> columns, int batchSize,
+ List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites, boolean isolated,
+ long readaheadThreshold) throws NotServingTabletException, ThriftSecurityException, org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
+
+ String tableId = new String(textent.getTable(), Constants.UTF8);
+ if (!security.canScan(credentials, tableId, Tables.getNamespaceId(instance, tableId), range, columns, ssiList, ssio, authorizations))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ if (!security.userHasAuthorizations(credentials, authorizations))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
+
+ KeyExtent extent = new KeyExtent(textent);
+
+ // wait for any writes that are in flight.. this done to ensure
+ // consistency across client restarts... assume a client writes
+ // to accumulo and dies while waiting for a confirmation from
+ // accumulo... the client process restarts and tries to read
+ // data from accumulo making the assumption that it will get
+ // any writes previously made... however if the server side thread
+ // processing the write from the dead client is still in progress,
+ // the restarted client may not see the write unless we wait here.
+ // this behavior is very important when the client is reading the
+ // metadata
+ if (waitForWrites)
+ writeTracker.waitForWrites(TabletType.type(extent));
+
+ Tablet tablet = onlineTablets.get(extent);
+ if (tablet == null)
+ throw new NotServingTabletException(textent);
+
+ ScanSession scanSession = new ScanSession();
+ scanSession.user = credentials.getPrincipal();
+ scanSession.extent = new KeyExtent(extent);
+ scanSession.columnSet = new HashSet<Column>();
+ scanSession.ssiList = ssiList;
+ scanSession.ssio = ssio;
+ scanSession.auths = new Authorizations(authorizations);
+ scanSession.interruptFlag = new AtomicBoolean();
+ scanSession.readaheadThreshold = readaheadThreshold;
+
+ for (TColumn tcolumn : columns) {
+ scanSession.columnSet.add(new Column(tcolumn));
+ }
+
+ scanSession.scanner = tablet.createScanner(new Range(range), batchSize, scanSession.columnSet, scanSession.auths, ssiList, ssio, isolated,
+ scanSession.interruptFlag);
+
+ long sid = sessionManager.createSession(scanSession, true);
+
+ ScanResult scanResult;
+ try {
+ scanResult = continueScan(tinfo, sid, scanSession);
+ } catch (NoSuchScanIDException e) {
+ log.error("The impossible happened", e);
+ throw new RuntimeException();
+ } finally {
+ sessionManager.unreserveSession(sid);
+ }
+
+ return new InitialScan(sid, scanResult);
+ }
+
+ @Override
+ public ScanResult continueScan(TInfo tinfo, long scanID) throws NoSuchScanIDException, NotServingTabletException,
+ org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
+ ScanSession scanSession = (ScanSession) sessionManager.reserveSession(scanID);
+ if (scanSession == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ try {
+ return continueScan(tinfo, scanID, scanSession);
+ } finally {
+ sessionManager.unreserveSession(scanSession);
+ }
+ }
+
+ private ScanResult continueScan(TInfo tinfo, long scanID, ScanSession scanSession) throws NoSuchScanIDException, NotServingTabletException,
+ org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
+
+ if (scanSession.nextBatchTask == null) {
+ scanSession.nextBatchTask = new NextBatchTask(scanID, scanSession.interruptFlag);
+ resourceManager.executeReadAhead(scanSession.extent, scanSession.nextBatchTask);
+ }
+
+ ScanBatch bresult;
+ try {
+ bresult = scanSession.nextBatchTask.get(MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS, TimeUnit.MILLISECONDS);
+ scanSession.nextBatchTask = null;
+ } catch (ExecutionException e) {
+ sessionManager.removeSession(scanID);
+ if (e.getCause() instanceof NotServingTabletException)
+ throw (NotServingTabletException) e.getCause();
+ else if (e.getCause() instanceof TooManyFilesException)
+ throw new org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException(scanSession.extent.toThrift());
+ else
+ throw new RuntimeException(e);
+ } catch (CancellationException ce) {
+ sessionManager.removeSession(scanID);
+ Tablet tablet = onlineTablets.get(scanSession.extent);
+ if (tablet == null || tablet.isClosed())
+ throw new NotServingTabletException(scanSession.extent.toThrift());
+ else
+ throw new NoSuchScanIDException();
+ } catch (TimeoutException e) {
+ List<TKeyValue> param = Collections.emptyList();
+ long timeout = acuConf.getTimeInMillis(Property.TSERV_CLIENT_TIMEOUT);
+ sessionManager.removeIfNotAccessed(scanID, timeout);
+ return new ScanResult(param, true);
+ } catch (Throwable t) {
+ sessionManager.removeSession(scanID);
+ log.warn("Failed to get next batch", t);
+ throw new RuntimeException(t);
+ }
+
+ ScanResult scanResult = new ScanResult(Key.compress(bresult.results), bresult.more);
+
+ scanSession.entriesReturned += scanResult.results.size();
+
+ scanSession.batchCount++;
+
+ if (scanResult.more && scanSession.batchCount > scanSession.readaheadThreshold) {
+ // start reading next batch while current batch is transmitted
+ // to client
+ scanSession.nextBatchTask = new NextBatchTask(scanID, scanSession.interruptFlag);
+ resourceManager.executeReadAhead(scanSession.extent, scanSession.nextBatchTask);
+ }
+
+ if (!scanResult.more)
+ closeScan(tinfo, scanID);
+
+ return scanResult;
+ }
+
+ @Override
+ public void closeScan(TInfo tinfo, long scanID) {
+ ScanSession ss = (ScanSession) sessionManager.removeSession(scanID);
+ if (ss != null) {
+ long t2 = System.currentTimeMillis();
+
+ log.debug(String.format("ScanSess tid %s %s %,d entries in %.2f secs, nbTimes = [%s] ", TServerUtils.clientAddress.get(), ss.extent.getTableId()
+ .toString(), ss.entriesReturned, (t2 - ss.startTime) / 1000.0, ss.nbTimes.toString()));
+ if (scanMetrics.isEnabled()) {
+ scanMetrics.add(TabletServerScanMetrics.scan, t2 - ss.startTime);
+ scanMetrics.add(TabletServerScanMetrics.resultSize, ss.entriesReturned);
+ }
+ }
+ }
+
+ @Override
+ public InitialMultiScan startMultiScan(TInfo tinfo, TCredentials credentials, Map<TKeyExtent,List<TRange>> tbatch, List<TColumn> tcolumns,
+ List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites) throws ThriftSecurityException {
+ // find all of the tables that need to be scanned
+ HashSet<String> tables = new HashSet<String>();
+ for (TKeyExtent keyExtent : tbatch.keySet()) {
+ tables.add(new String(keyExtent.getTable(), Constants.UTF8));
+ }
+
+ if (tables.size() != 1)
+ throw new IllegalArgumentException("Cannot batch scan over multiple tables");
+
+ // check if user has permission to the tables
+ for (String tableId : tables)
+ if (!security.canScan(credentials, tableId, Tables.getNamespaceId(instance, tableId), tbatch, tcolumns, ssiList, ssio, authorizations))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ try {
+ if (!security.userHasAuthorizations(credentials, authorizations))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
+ } catch (ThriftSecurityException tse) {
+ log.error(tse, tse);
+ throw tse;
+ }
+ Map<KeyExtent,List<Range>> batch = Translator.translate(tbatch, new TKeyExtentTranslator(), new Translator.ListTranslator<TRange,Range>(
+ new TRangeTranslator()));
+
+ // This is used to determine which thread pool to use
+ KeyExtent threadPoolExtent = batch.keySet().iterator().next();
+
+ if (waitForWrites)
+ writeTracker.waitForWrites(TabletType.type(batch.keySet()));
+
+ MultiScanSession mss = new MultiScanSession();
+ mss.user = credentials.getPrincipal();
+ mss.queries = batch;
+ mss.columnSet = new HashSet<Column>(tcolumns.size());
+ mss.ssiList = ssiList;
+ mss.ssio = ssio;
+ mss.auths = new Authorizations(authorizations);
+
+ mss.numTablets = batch.size();
+ for (List<Range> ranges : batch.values()) {
+ mss.numRanges += ranges.size();
+ }
+
+ for (TColumn tcolumn : tcolumns)
+ mss.columnSet.add(new Column(tcolumn));
+
+ mss.threadPoolExtent = threadPoolExtent;
+
+ long sid = sessionManager.createSession(mss, true);
+
+ MultiScanResult result;
+ try {
+ result = continueMultiScan(tinfo, sid, mss);
+ } catch (NoSuchScanIDException e) {
+ log.error("the impossible happened", e);
+ throw new RuntimeException("the impossible happened", e);
+ } finally {
+ sessionManager.unreserveSession(sid);
+ }
+
+ return new InitialMultiScan(sid, result);
+ }
+
+ @Override
+ public MultiScanResult continueMultiScan(TInfo tinfo, long scanID) throws NoSuchScanIDException {
+
+ MultiScanSession session = (MultiScanSession) sessionManager.reserveSession(scanID);
+
+ if (session == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ try {
+ return continueMultiScan(tinfo, scanID, session);
+ } finally {
+ sessionManager.unreserveSession(session);
+ }
+ }
+
+ private MultiScanResult continueMultiScan(TInfo tinfo, long scanID, MultiScanSession session) throws NoSuchScanIDException {
+
+ if (session.lookupTask == null) {
+ session.lookupTask = new LookupTask(scanID);
+ resourceManager.executeReadAhead(session.threadPoolExtent, session.lookupTask);
+ }
+
+ try {
+ MultiScanResult scanResult = session.lookupTask.get(MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS, TimeUnit.MILLISECONDS);
+ session.lookupTask = null;
+ return scanResult;
+ } catch (TimeoutException e1) {
+ long timeout = acuConf.getTimeInMillis(Property.TSERV_CLIENT_TIMEOUT);
+ sessionManager.removeIfNotAccessed(scanID, timeout);
+ List<TKeyValue> results = Collections.emptyList();
+ Map<TKeyExtent,List<TRange>> failures = Collections.emptyMap();
+ List<TKeyExtent> fullScans = Collections.emptyList();
+ return new MultiScanResult(results, failures, fullScans, null, null, false, true);
+ } catch (Throwable t) {
+ sessionManager.removeSession(scanID);
+ log.warn("Failed to get multiscan result", t);
+ throw new RuntimeException(t);
+ }
+ }
+
+ @Override
+ public void closeMultiScan(TInfo tinfo, long scanID) throws NoSuchScanIDException {
+ MultiScanSession session = (MultiScanSession) sessionManager.removeSession(scanID);
+ if (session == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ long t2 = System.currentTimeMillis();
+ log.debug(String.format("MultiScanSess %s %,d entries in %.2f secs (lookup_time:%.2f secs tablets:%,d ranges:%,d) ", TServerUtils.clientAddress.get(),
+ session.numEntries, (t2 - session.startTime) / 1000.0, session.totalLookupTime / 1000.0, session.numTablets, session.numRanges));
+ }
+
+ @Override
+ public long startUpdate(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException {
+ // Make sure user is real
+
+ security.authenticateUser(credentials, credentials);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
+
+ UpdateSession us = new UpdateSession();
+ us.violations = new Violations();
+ us.credentials = credentials;
+ us.cenv = new TservConstraintEnv(security, us.credentials);
+
+ long sid = sessionManager.createSession(us, false);
+
+ return sid;
+ }
+
+ private void setUpdateTablet(UpdateSession us, KeyExtent keyExtent) {
+ long t1 = System.currentTimeMillis();
+ if (us.currentTablet != null && us.currentTablet.getExtent().equals(keyExtent))
+ return;
+ if (us.currentTablet == null && (us.failures.containsKey(keyExtent) || us.authFailures.containsKey(keyExtent))) {
+ // if there were previous failures, then do not accept additional writes
+ return;
+ }
+
+ try {
+ // if user has no permission to write to this table, add it to
+ // the failures list
+ boolean sameTable = us.currentTablet != null && (us.currentTablet.getExtent().getTableId().equals(keyExtent.getTableId()));
+ String tableId = keyExtent.getTableId().toString();
+ if (sameTable || security.canWrite(us.credentials, tableId, Tables.getNamespaceId(instance, tableId))) {
+ long t2 = System.currentTimeMillis();
+ us.authTimes.addStat(t2 - t1);
+ us.currentTablet = onlineTablets.get(keyExtent);
+ if (us.currentTablet != null) {
+ us.queuedMutations.put(us.currentTablet, new ArrayList<Mutation>());
+ } else {
+ // not serving tablet, so report all mutations as
+ // failures
+ us.failures.put(keyExtent, 0l);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.unknownTabletErrors, 0);
+ }
+ } else {
+ log.warn("Denying access to table " + keyExtent.getTableId() + " for user " + us.credentials.getPrincipal());
+ long t2 = System.currentTimeMillis();
+ us.authTimes.addStat(t2 - t1);
+ us.currentTablet = null;
+ us.authFailures.put(keyExtent, SecurityErrorCode.PERMISSION_DENIED);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
+ return;
+ }
+ } catch (ThriftSecurityException e) {
+ log.error("Denying permission to check user " + us.credentials.getPrincipal() + " with user " + e.getUser(), e);
+ long t2 = System.currentTimeMillis();
+ us.authTimes.addStat(t2 - t1);
+ us.currentTablet = null;
+ us.authFailures.put(keyExtent, e.getCode());
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
+ return;
+ }
+ }
+
+ @Override
+ public void applyUpdates(TInfo tinfo, long updateID, TKeyExtent tkeyExtent, List<TMutation> tmutations) {
+ UpdateSession us = (UpdateSession) sessionManager.reserveSession(updateID);
+ if (us == null) {
+ throw new RuntimeException("No Such SessionID");
+ }
+
+ try {
+ KeyExtent keyExtent = new KeyExtent(tkeyExtent);
+ setUpdateTablet(us, keyExtent);
+
+ if (us.currentTablet != null) {
+ List<Mutation> mutations = us.queuedMutations.get(us.currentTablet);
+ for (TMutation tmutation : tmutations) {
+ Mutation mutation = new ServerMutation(tmutation);
+ mutations.add(mutation);
+ us.queuedMutationSize += mutation.numBytes();
+ }
+ if (us.queuedMutationSize > getSystemConfiguration().getMemoryInBytes(Property.TSERV_MUTATION_QUEUE_MAX))
+ flush(us);
+ }
+ } finally {
+ sessionManager.unreserveSession(us);
+ }
+ }
+
+ private void flush(UpdateSession us) {
+
+ int mutationCount = 0;
+ Map<CommitSession,List<Mutation>> sendables = new HashMap<CommitSession,List<Mutation>>();
+ Throwable error = null;
+
+ long pt1 = System.currentTimeMillis();
+
+ boolean containsMetadataTablet = false;
+ for (Tablet tablet : us.queuedMutations.keySet())
+ if (tablet.getExtent().isMeta())
+ containsMetadataTablet = true;
+
+ if (!containsMetadataTablet && us.queuedMutations.size() > 0)
+ TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
+
+ Span prep = Trace.start("prep");
+ try {
+ for (Entry<Tablet,? extends List<Mutation>> entry : us.queuedMutations.entrySet()) {
+
+ Tablet tablet = entry.getKey();
+ List<Mutation> mutations = entry.getValue();
+ if (mutations.size() > 0) {
+ try {
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.mutationArraySize, mutations.size());
+
+ CommitSession commitSession = tablet.prepareMutationsForCommit(us.cenv, mutations);
+ if (commitSession == null) {
+ if (us.currentTablet == tablet) {
+ us.currentTablet = null;
+ }
+ us.failures.put(tablet.getExtent(), us.successfulCommits.get(tablet));
+ } else {
+ sendables.put(commitSession, mutations);
+ mutationCount += mutations.size();
+ }
+
+ } catch (TConstraintViolationException e) {
+ us.violations.add(e.getViolations());
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.constraintViolations, 0);
+
+ if (e.getNonViolators().size() > 0) {
+ // only log and commit mutations if there were some
+ // that did not
+ // violate constraints... this is what
+ // prepareMutationsForCommit()
+ // expects
+ sendables.put(e.getCommitSession(), e.getNonViolators());
+ }
+
+ mutationCount += mutations.size();
+
+ } catch (HoldTimeoutException t) {
+ error = t;
+ log.debug("Giving up on mutations due to a long memory hold time");
+ break;
+ } catch (Throwable t) {
+ error = t;
+ log.error("Unexpected error preparing for commit", error);
+ break;
+ }
+ }
+ }
+ } finally {
+ prep.stop();
+ }
+
+ long pt2 = System.currentTimeMillis();
+ us.prepareTimes.addStat(pt2 - pt1);
+ updateAvgPrepTime(pt2 - pt1, us.queuedMutations.size());
+
+ if (error != null) {
+ for (Entry<CommitSession,List<Mutation>> e : sendables.entrySet()) {
+ e.getKey().abortCommit(e.getValue());
+ }
+ throw new RuntimeException(error);
+ }
+ try {
+ Span wal = Trace.start("wal");
+ try {
+ while (true) {
+ try {
+ long t1 = System.currentTimeMillis();
+
+ logger.logManyTablets(sendables);
+
+ long t2 = System.currentTimeMillis();
+ us.walogTimes.addStat(t2 - t1);
+ updateWalogWriteTime((t2 - t1));
+ break;
+ } catch (IOException ex) {
+ log.warn("logging mutations failed, retrying");
+ } catch (FSError ex) { // happens when DFS is localFS
+ log.warn("logging mutations failed, retrying");
+ } catch (Throwable t) {
+ log.error("Unknown exception logging mutations, counts for mutations in flight not decremented!", t);
+ throw new RuntimeException(t);
+ }
+ }
+ } finally {
+ wal.stop();
+ }
+
+ Span commit = Trace.start("commit");
+ try {
+ long t1 = System.currentTimeMillis();
+ for (Entry<CommitSession,? extends List<Mutation>> entry : sendables.entrySet()) {
+ CommitSession commitSession = entry.getKey();
+ List<Mutation> mutations = entry.getValue();
+
+ commitSession.commit(mutations);
+
+ Tablet tablet = commitSession.getTablet();
+
+ if (tablet == us.currentTablet) {
+ // because constraint violations may filter out some
+ // mutations, for proper
+ // accounting with the client code, need to increment
+ // the count based
+ // on the original number of mutations from the client
+ // NOT the filtered number
+ us.successfulCommits.increment(tablet, us.queuedMutations.get(tablet).size());
+ }
+ }
+ long t2 = System.currentTimeMillis();
+
+ us.flushTime += (t2 - pt1);
+ us.commitTimes.addStat(t2 - t1);
+
+ updateAvgCommitTime(t2 - t1, sendables.size());
+ } finally {
+ commit.stop();
+ }
+ } finally {
+ us.queuedMutations.clear();
+ if (us.currentTablet != null) {
+ us.queuedMutations.put(us.currentTablet, new ArrayList<Mutation>());
+ }
+ us.queuedMutationSize = 0;
+ }
+ us.totalUpdates += mutationCount;
+ }
+
+ private void updateWalogWriteTime(long time) {
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.waLogWriteTime, time);
+ }
+
+ private void updateAvgCommitTime(long time, int size) {
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.commitTime, (long) ((time) / (double) size));
+ }
+
+ private void updateAvgPrepTime(long time, int size) {
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.commitPrep, (long) ((time) / (double) size));
+ }
+
+ @Override
+ public UpdateErrors closeUpdate(TInfo tinfo, long updateID) throws NoSuchScanIDException {
+ UpdateSession us = (UpdateSession) sessionManager.removeSession(updateID);
+ if (us == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ // clients may or may not see data from an update session while
+ // it is in progress, however when the update session is closed
+ // want to ensure that reads wait for the write to finish
+ long opid = writeTracker.startWrite(us.queuedMutations.keySet());
+
+ try {
+ flush(us);
+ } finally {
+ writeTracker.finishWrite(opid);
+ }
+
+ log.debug(String.format("UpSess %s %,d in %.3fs, at=[%s] ft=%.3fs(pt=%.3fs lt=%.3fs ct=%.3fs)", TServerUtils.clientAddress.get(), us.totalUpdates,
+ (System.currentTimeMillis() - us.startTime) / 1000.0, us.authTimes.toString(), us.flushTime / 1000.0, us.prepareTimes.getSum() / 1000.0,
+ us.walogTimes.getSum() / 1000.0, us.commitTimes.getSum() / 1000.0));
+ if (us.failures.size() > 0) {
+ Entry<KeyExtent,Long> first = us.failures.entrySet().iterator().next();
+ log.debug(String.format("Failures: %d, first extent %s successful commits: %d", us.failures.size(), first.getKey().toString(), first.getValue()));
+ }
+ List<ConstraintViolationSummary> violations = us.violations.asList();
+ if (violations.size() > 0) {
+ ConstraintViolationSummary first = us.violations.asList().iterator().next();
+ log.debug(String.format("Violations: %d, first %s occurs %d", violations.size(), first.violationDescription, first.numberOfViolatingMutations));
+ }
+ if (us.authFailures.size() > 0) {
+ KeyExtent first = us.authFailures.keySet().iterator().next();
+ log.debug(String.format("Authentication Failures: %d, first %s", us.authFailures.size(), first.toString()));
+ }
+
+ return new UpdateErrors(Translator.translate(us.failures, Translators.KET), Translator.translate(violations, Translators.CVST), Translator.translate(
+ us.authFailures, Translators.KET));
+ }
+
+ @Override
+ public void update(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, TMutation tmutation) throws NotServingTabletException,
+ ConstraintViolationException, ThriftSecurityException {
+
+ String tableId = new String(tkeyExtent.getTable(), Constants.UTF8);
+ if (!security.canWrite(credentials, tableId, Tables.getNamespaceId(instance, tableId)))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ KeyExtent keyExtent = new KeyExtent(tkeyExtent);
+ Tablet tablet = onlineTablets.get(new KeyExtent(keyExtent));
+ if (tablet == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+
+ if (!keyExtent.isMeta())
+ TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
+
+ long opid = writeTracker.startWrite(TabletType.type(keyExtent));
+
+ try {
+ Mutation mutation = new ServerMutation(tmutation);
+ List<Mutation> mutations = Collections.singletonList(mutation);
+
+ Span prep = Trace.start("prep");
+ CommitSession cs;
+ try {
+ cs = tablet.prepareMutationsForCommit(new TservConstraintEnv(security, credentials), mutations);
+ } finally {
+ prep.stop();
+ }
+ if (cs == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+
+ while (true) {
+ try {
+ Span wal = Trace.start("wal");
+ try {
+ logger.log(cs, cs.getWALogSeq(), mutation);
+ } finally {
+ wal.stop();
+ }
+ break;
+ } catch (IOException ex) {
+ log.warn(ex, ex);
+ }
+ }
+
+ Span commit = Trace.start("commit");
+ try {
+ cs.commit(mutations);
+ } finally {
+ commit.stop();
+ }
+ } catch (TConstraintViolationException e) {
+ throw new ConstraintViolationException(Translator.translate(e.getViolations().asList(), Translators.CVST));
+ } finally {
+ writeTracker.finishWrite(opid);
+ }
+ }
+
+ private void checkConditions(Map<KeyExtent,List<ServerConditionalMutation>> updates, ArrayList<TCMResult> results, ConditionalSession cs,
+ List<String> symbols) throws IOException {
+ Iterator<Entry<KeyExtent,List<ServerConditionalMutation>>> iter = updates.entrySet().iterator();
+
+ CompressedIterators compressedIters = new CompressedIterators(symbols);
+
+ while (iter.hasNext()) {
+ Entry<KeyExtent,List<ServerConditionalMutation>> entry = iter.next();
+ Tablet tablet = onlineTablets.get(entry.getKey());
+
+ if (tablet == null || tablet.isClosed()) {
+ for (ServerConditionalMutation scm : entry.getValue())
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ iter.remove();
+ } else {
+ List<ServerConditionalMutation> okMutations = new ArrayList<ServerConditionalMutation>(entry.getValue().size());
+
+ for (ServerConditionalMutation scm : entry.getValue()) {
+ if (checkCondition(results, cs, compressedIters, tablet, scm))
+ okMutations.add(scm);
+ }
+
+ entry.setValue(okMutations);
+ }
+
+ }
+ }
+
+ boolean checkCondition(ArrayList<TCMResult> results, ConditionalSession cs, CompressedIterators compressedIters, Tablet tablet,
+ ServerConditionalMutation scm) throws IOException {
+ boolean add = true;
+
+ Set<Column> emptyCols = Collections.emptySet();
+
+ for (TCondition tc : scm.getConditions()) {
+
+ Range range;
+ if (tc.hasTimestamp)
+ range = Range.exact(new Text(scm.getRow()), new Text(tc.getCf()), new Text(tc.getCq()), new Text(tc.getCv()), tc.getTs());
+ else
+ range = Range.exact(new Text(scm.getRow()), new Text(tc.getCf()), new Text(tc.getCq()), new Text(tc.getCv()));
+
+ IterConfig ic = compressedIters.decompress(tc.iterators);
+
+ Scanner scanner = tablet.createScanner(range, 1, emptyCols, cs.auths, ic.ssiList, ic.ssio, false, cs.interruptFlag);
+
+ try {
+ ScanBatch batch = scanner.read();
+
+ Value val = null;
+
+ for (KVEntry entry2 : batch.results) {
+ val = entry2.getValue();
+ break;
+ }
+
+ if ((val == null ^ tc.getVal() == null) || (val != null && !Arrays.equals(tc.getVal(), val.get()))) {
+ results.add(new TCMResult(scm.getID(), TCMStatus.REJECTED));
+ add = false;
+ break;
+ }
+
+ } catch (TabletClosedException e) {
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ add = false;
+ break;
+ } catch (IterationInterruptedException iie) {
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ add = false;
+ break;
+ } catch (TooManyFilesException tmfe) {
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ add = false;
+ break;
+ }
+ }
+ return add;
+ }
+
+ private void writeConditionalMutations(Map<KeyExtent,List<ServerConditionalMutation>> updates, ArrayList<TCMResult> results, ConditionalSession sess) {
+ Set<Entry<KeyExtent,List<ServerConditionalMutation>>> es = updates.entrySet();
+
+ Map<CommitSession,List<Mutation>> sendables = new HashMap<CommitSession,List<Mutation>>();
+
+ boolean sessionCanceled = sess.interruptFlag.get();
+
+ Span prepSpan = Trace.start("prep");
+ try {
+ long t1 = System.currentTimeMillis();
+ for (Entry<KeyExtent,List<ServerConditionalMutation>> entry : es) {
+ Tablet tablet = onlineTablets.get(entry.getKey());
+ if (tablet == null || tablet.isClosed() || sessionCanceled) {
+ for (ServerConditionalMutation scm : entry.getValue())
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ } else {
+ try {
+
+ @SuppressWarnings("unchecked")
+ List<Mutation> mutations = (List<Mutation>) (List<? extends Mutation>) entry.getValue();
+ if (mutations.size() > 0) {
+
+ CommitSession cs = tablet.prepareMutationsForCommit(new TservConstraintEnv(security, sess.credentials), mutations);
+
+ if (cs == null) {
+ for (ServerConditionalMutation scm : entry.getValue())
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ } else {
+ for (ServerConditionalMutation scm : entry.getValue())
+ results.add(new TCMResult(scm.getID(), TCMStatus.ACCEPTED));
+ sendables.put(cs, mutations);
+ }
+ }
+ } catch (TConstraintViolationException e) {
+ if (e.getNonViolators().size() > 0) {
+ sendables.put(e.getCommitSession(), e.getNonViolators());
+ for (Mutation m : e.getNonViolators())
+ results.add(new TCMResult(((ServerConditionalMutation) m).getID(), TCMStatus.ACCEPTED));
+ }
+
+ for (Mutation m : e.getViolators())
+ results.add(new TCMResult(((ServerConditionalMutation) m).getID(), TCMStatus.VIOLATED));
+ }
+ }
+ }
+
+ long t2 = System.currentTimeMillis();
+ updateAvgPrepTime(t2 - t1, es.size());
+ } finally {
+ prepSpan.stop();
+ }
+
+ Span walSpan = Trace.start("wal");
+ try {
+ while (true && sendables.size() > 0) {
+ try {
+ long t1 = System.currentTimeMillis();
+ logger.logManyTablets(sendables);
+ long t2 = System.currentTimeMillis();
+ updateWalogWriteTime(t2 - t1);
+ break;
+ } catch (IOException ex) {
+ log.warn("logging mutations failed, retrying");
+ } catch (FSError ex) { // happens when DFS is localFS
+ log.warn("logging mutations failed, retrying");
+ } catch (Throwable t) {
+ log.error("Unknown exception logging mutations, counts for mutations in flight not decremented!", t);
+ throw new RuntimeException(t);
+ }
+ }
+ } finally {
+ walSpan.stop();
+ }
+
+ Span commitSpan = Trace.start("commit");
+ try {
+ long t1 = System.currentTimeMillis();
+ for (Entry<CommitSession,? extends List<Mutation>> entry : sendables.entrySet()) {
+ CommitSession commitSession = entry.getKey();
+ List<Mutation> mutations = entry.getValue();
+
+ commitSession.commit(mutations);
+ }
+ long t2 = System.currentTimeMillis();
+ updateAvgCommitTime(t2 - t1, sendables.size());
+ } finally {
+ commitSpan.stop();
+ }
+
+ }
+
+ private Map<KeyExtent,List<ServerConditionalMutation>> conditionalUpdate(ConditionalSession cs, Map<KeyExtent,List<ServerConditionalMutation>> updates,
+ ArrayList<TCMResult> results, List<String> symbols) throws IOException {
+ // sort each list of mutations, this is done to avoid deadlock and doing seeks in order is more efficient and detect duplicate rows.
+ ConditionalMutationSet.sortConditionalMutations(updates);
+
+ Map<KeyExtent,List<ServerConditionalMutation>> deferred = new HashMap<KeyExtent,List<ServerConditionalMutation>>();
+
+ // can not process two mutations for the same row, because one will not see what the other writes
+ ConditionalMutationSet.deferDuplicatesRows(updates, deferred);
+
+ // get as many locks as possible w/o blocking... defer any rows that are locked
+ List<RowLock> locks = rowLocks.acquireRowlocks(updates, deferred);
+ try {
+ Span checkSpan = Trace.start("Check conditions");
+ try {
+ checkConditions(updates, results, cs, symbols);
+ } finally {
+ checkSpan.stop();
+ }
+
+ Span updateSpan = Trace.start("apply conditional mutations");
+ try {
+ writeConditionalMutations(updates, results, cs);
+ } finally {
+ updateSpan.stop();
+ }
+ } finally {
+ rowLocks.releaseRowLocks(locks);
+ }
+ return deferred;
+ }
+
+ @Override
+ public TConditionalSession startConditionalUpdate(TInfo tinfo, TCredentials credentials, List<ByteBuffer> authorizations, String tableId)
+ throws ThriftSecurityException, TException {
+
+ Authorizations userauths = null;
+ if (!security.canConditionallyUpdate(credentials, tableId, Tables.getNamespaceId(instance, tableId), authorizations))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ userauths = security.getUserAuthorizations(credentials);
+ for (ByteBuffer auth : authorizations)
+ if (!userauths.contains(ByteBufferUtil.toBytes(auth)))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
+
+ ConditionalSession cs = new ConditionalSession();
+ cs.auths = new Authorizations(authorizations);
+ cs.credentials = credentials;
+ cs.tableId = tableId;
+ cs.interruptFlag = new AtomicBoolean();
+
+ long sid = sessionManager.createSession(cs, false);
+ return new TConditionalSession(sid, lockID, sessionManager.getMaxIdleTime());
+ }
+
+ @Override
+ public List<TCMResult> conditionalUpdate(TInfo tinfo, long sessID, Map<TKeyExtent,List<TConditionalMutation>> mutations, List<String> symbols)
+ throws NoSuchScanIDException, TException {
+
+ ConditionalSession cs = (ConditionalSession) sessionManager.reserveSession(sessID);
+
+ if (cs == null || cs.interruptFlag.get())
+ throw new NoSuchScanIDException();
+
+ if (!cs.tableId.equals(MetadataTable.ID) && !cs.tableId.equals(RootTable.ID))
+ TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
+
+ Text tid = new Text(cs.tableId);
+ long opid = writeTracker.startWrite(TabletType.type(new KeyExtent(tid, null, null)));
+
+ try {
+ Map<KeyExtent,List<ServerConditionalMutation>> updates = Translator.translate(mutations, Translators.TKET,
+ new Translator.ListTranslator<TConditionalMutation,ServerConditionalMutation>(ServerConditionalMutation.TCMT));
+
+ for (KeyExtent ke : updates.keySet())
+ if (!ke.getTableId().equals(tid))
+ throw new IllegalArgumentException("Unexpected table id " + tid + " != " + ke.getTableId());
+
+ ArrayList<TCMResult> results = new ArrayList<TCMResult>();
+
+ Map<KeyExtent,List<ServerConditionalMutation>> deferred = conditionalUpdate(cs, updates, results, symbols);
+
+ while (deferred.size() > 0) {
+ deferred = conditionalUpdate(cs, deferred, results, symbols);
+ }
+
+ return results;
+ } catch (IOException ioe) {
+ throw new TException(ioe);
+ } finally {
+ writeTracker.finishWrite(opid);
+ sessionManager.unreserveSession(sessID);
+ }
+ }
+
+ @Override
+ public void invalidateConditionalUpdate(TInfo tinfo, long sessID) throws TException {
+ // this method should wait for any running conditional update to complete
+ // after this method returns a conditional update should not be able to start
+
+ ConditionalSession cs = (ConditionalSession) sessionManager.getSession(sessID);
+ if (cs != null)
+ cs.interruptFlag.set(true);
+
+ cs = (ConditionalSession) sessionManager.reserveSession(sessID, true);
+ if (cs != null)
+ sessionManager.removeSession(sessID, true);
+ }
+
+ @Override
+ public void closeConditionalUpdate(TInfo tinfo, long sessID) throws TException {
+ sessionManager.removeSession(sessID, false);
+ }
+
+ @Override
+ public void splitTablet(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, ByteBuffer splitPoint) throws NotServingTabletException,
+ ThriftSecurityException {
+
+ String tableId = new String(ByteBufferUtil.toBytes(tkeyExtent.table));
+ String namespaceId;
+ try {
+ namespaceId = Tables.getNamespaceId(instance, tableId);
+ } catch (IllegalArgumentException ex) {
+ // table does not exist, try to educate the client
+ throw new NotServingTabletException(tkeyExtent);
+ }
-
++
+ if (!security.canSplitTablet(credentials, tableId, namespaceId))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ KeyExtent keyExtent = new KeyExtent(tkeyExtent);
+
+ Tablet tablet = onlineTablets.get(keyExtent);
+ if (tablet == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+
+ if (keyExtent.getEndRow() == null || !keyExtent.getEndRow().equals(ByteBufferUtil.toText(splitPoint))) {
+ try {
+ if (TabletServer.this.splitTablet(tablet, ByteBufferUtil.toBytes(splitPoint)) == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+ } catch (IOException e) {
+ log.warn("Failed to split " + keyExtent, e);
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Override
+ public TabletServerStatus getTabletServerStatus(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ return getStats(sessionManager.getActiveScansPerTable());
+ }
+
+ @Override
+ public List<TabletStats> getTabletStats(TInfo tinfo, TCredentials credentials, String tableId) throws ThriftSecurityException, TException {
+ TreeMap<KeyExtent,Tablet> onlineTabletsCopy;
+ synchronized (onlineTablets) {
+ onlineTabletsCopy = new TreeMap<KeyExtent,Tablet>(onlineTablets);
+ }
+ List<TabletStats> result = new ArrayList<TabletStats>();
+ Text text = new Text(tableId);
+ KeyExtent start = new KeyExtent(text, new Text(), null);
+ for (Entry<KeyExtent,Tablet> entry : onlineTabletsCopy.tailMap(start).entrySet()) {
+ KeyExtent ke = entry.getKey();
+ if (ke.getTableId().compareTo(text) == 0) {
+ Tablet tablet = entry.getValue();
+ TabletStats stats = tablet.timer.getTabletStats();
+ stats.extent = ke.toThrift();
+ stats.ingestRate = tablet.ingestRate();
+ stats.queryRate = tablet.queryRate();
+ stats.splitCreationTime = tablet.getSplitCreationTime();
+ stats.numEntries = tablet.getNumEntries();
+ result.add(stats);
+ }
+ }
+ return result;
+ }
+
+ private ZooCache masterLockCache = new ZooCache();
+
+ private void checkPermission(TCredentials credentials, String lock, final String request) throws ThriftSecurityException {
+ boolean fatal = false;
+ try {
+ log.debug("Got " + request + " message from user: " + credentials.getPrincipal());
+ if (!security.canPerformSystemActions(credentials)) {
+ log.warn("Got " + request + " message from user: " + credentials.getPrincipal());
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ }
+ } catch (ThriftSecurityException e) {
+ log.warn("Got " + request + " message from unauthenticatable user: " + e.getUser());
+ if (SystemCredentials.get().getToken().getClass().getName().equals(credentials.getTokenClassName())) {
+ log.fatal("Got message from a service with a mismatched configuration. Please ensure a compatible configuration.", e);
+ fatal = true;
+ }
+ throw e;
+ } finally {
+ if (fatal) {
+ Halt.halt(1, new Runnable() {
+ @Override
+ public void run() {
+ logGCInfo(getSystemConfiguration());
+ }
+ });
+ }
+ }
+
+ if (tabletServerLock == null || !tabletServerLock.wasLockAcquired()) {
+ log.warn("Got " + request + " message from master before lock acquired, ignoring...");
+ throw new RuntimeException("Lock not acquired");
+ }
+
+ if (tabletServerLock != null && tabletServerLock.wasLockAcquired() && !tabletServerLock.isLocked()) {
+ Halt.halt(1, new Runnable() {
+ @Override
+ public void run() {
+ log.info("Tablet server no longer holds lock during checkPermission(
<TRUNCATED>
[30/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
index 9c02219,0000000..1f83541
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
@@@ -1,681 -1,0 +1,680 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.mapreduce.lib.util.OutputConfigurator;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.SecurityErrorCode;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * This class allows MapReduce jobs to use Accumulo as the sink for data. This {@link OutputFormat} accepts keys and values of type {@link Text} (for a table
+ * name) and {@link Mutation} from the Map and Reduce functions.
+ *
+ * The user must specify the following via static configurator methods:
+ *
+ * <ul>
+ * <li>{@link AccumuloOutputFormat#setConnectorInfo(Job, String, AuthenticationToken)}
+ * <li>{@link AccumuloOutputFormat#setZooKeeperInstance(Job, String, String)} OR {@link AccumuloOutputFormat#setMockInstance(Job, String)}
+ * </ul>
+ *
+ * Other static methods are optional.
+ */
+public class AccumuloOutputFormat extends OutputFormat<Text,Mutation> {
+
+ private static final Class<?> CLASS = AccumuloOutputFormat.class;
+ protected static final Logger log = Logger.getLogger(CLASS);
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
+ * conversion to a string, and is not intended to be secure.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission if {@link #setCreateTables(Job, boolean)} is set to true)
+ * @param token
+ * the user's password
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public static void setConnectorInfo(Job job, String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ OutputConfigurator.setConnectorInfo(CLASS, job.getConfiguration(), principal, token);
+ }
+
+ /**
+ * Determines if the connector has been configured.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the connector has been configured, false otherwise
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static Boolean isConnectorInfoSet(JobContext context) {
+ return OutputConfigurator.isConnectorInfoSet(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static String getPrincipal(JobContext context) {
+ return OutputConfigurator.getPrincipal(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Gets the serialized token class name from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static String getTokenClass(JobContext context) {
+ return OutputConfigurator.getTokenClass(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Gets the password from the configuration. WARNING: The password is stored in the Configuration and shared with all MapReduce tasks; It is BASE64 encoded to
+ * provide a charset safe conversion to a string, and is not intended to be secure.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the decoded user password
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static byte[] getToken(JobContext context) {
+ return OutputConfigurator.getToken(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Configures a {@link ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @param zooKeepers
+ * a comma-separated list of zookeeper servers
+ * @since 1.5.0
+ */
+ public static void setZooKeeperInstance(Job job, String instanceName, String zooKeepers) {
+ OutputConfigurator.setZooKeeperInstance(CLASS, job.getConfiguration(), instanceName, zooKeepers);
+ }
+
+ /**
+ * Configures a {@link MockInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @since 1.5.0
+ */
+ public static void setMockInstance(Job job, String instanceName) {
+ OutputConfigurator.setMockInstance(CLASS, job.getConfiguration(), instanceName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link Instance} based on the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return an Accumulo instance
+ * @since 1.5.0
+ * @see #setZooKeeperInstance(Job, String, String)
+ * @see #setMockInstance(Job, String)
+ */
+ protected static Instance getInstance(JobContext context) {
+ return OutputConfigurator.getInstance(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param level
+ * the logging level
+ * @since 1.5.0
+ */
+ public static void setLogLevel(Job job, Level level) {
+ OutputConfigurator.setLogLevel(CLASS, job.getConfiguration(), level);
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the log level
+ * @since 1.5.0
+ * @see #setLogLevel(Job, Level)
+ */
+ protected static Level getLogLevel(JobContext context) {
+ return OutputConfigurator.getLogLevel(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Sets the default table name to use if one emits a null in place of a table name for a given mutation. Table names can only be alpha-numeric and
+ * underscores.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.5.0
+ */
+ public static void setDefaultTableName(Job job, String tableName) {
+ OutputConfigurator.setDefaultTableName(CLASS, job.getConfiguration(), tableName);
+ }
+
+ /**
+ * Gets the default table name from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the default table name
+ * @since 1.5.0
+ * @see #setDefaultTableName(Job, String)
+ */
+ protected static String getDefaultTableName(JobContext context) {
+ return OutputConfigurator.getDefaultTableName(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Sets the configuration for for the job's {@link BatchWriter} instances. If not set, a new {@link BatchWriterConfig}, with sensible built-in defaults is
+ * used. Setting the configuration multiple times overwrites any previous configuration.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param bwConfig
+ * the configuration for the {@link BatchWriter}
+ * @since 1.5.0
+ */
+ public static void setBatchWriterOptions(Job job, BatchWriterConfig bwConfig) {
+ OutputConfigurator.setBatchWriterOptions(CLASS, job.getConfiguration(), bwConfig);
+ }
+
+ /**
+ * Gets the {@link BatchWriterConfig} settings.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the configuration object
+ * @since 1.5.0
+ * @see #setBatchWriterOptions(Job, BatchWriterConfig)
+ */
+ protected static BatchWriterConfig getBatchWriterOptions(JobContext context) {
+ return OutputConfigurator.getBatchWriterOptions(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Sets the directive to create new tables, as necessary. Table names can only be alpha-numeric and underscores.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setCreateTables(Job job, boolean enableFeature) {
+ OutputConfigurator.setCreateTables(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether tables are permitted to be created as needed.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the feature is disabled, false otherwise
+ * @since 1.5.0
+ * @see #setCreateTables(Job, boolean)
+ */
+ protected static Boolean canCreateTables(JobContext context) {
+ return OutputConfigurator.canCreateTables(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * Sets the directive to use simulation mode for this job. In simulation mode, no output is produced. This is useful for testing.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setSimulationMode(Job job, boolean enableFeature) {
+ OutputConfigurator.setSimulationMode(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether this feature is enabled.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setSimulationMode(Job, boolean)
+ */
+ protected static Boolean getSimulationMode(JobContext context) {
+ return OutputConfigurator.getSimulationMode(CLASS, InputFormatBase.getConfiguration(context));
+ }
+
+ /**
+ * A base class to be used to create {@link RecordWriter} instances that write to Accumulo.
+ */
+ protected static class AccumuloRecordWriter extends RecordWriter<Text,Mutation> {
+ private MultiTableBatchWriter mtbw = null;
+ private HashMap<Text,BatchWriter> bws = null;
+ private Text defaultTableName = null;
+
+ private boolean simulate = false;
+ private boolean createTables = false;
+
+ private long mutCount = 0;
+ private long valCount = 0;
+
+ private Connector conn;
+
+ protected AccumuloRecordWriter(TaskAttemptContext context) throws AccumuloException, AccumuloSecurityException, IOException {
+ Level l = getLogLevel(context);
+ if (l != null)
+ log.setLevel(getLogLevel(context));
+ this.simulate = getSimulationMode(context);
+ this.createTables = canCreateTables(context);
+
+ if (simulate)
+ log.info("Simulating output only. No writes to tables will occur");
+
+ this.bws = new HashMap<Text,BatchWriter>();
+
+ String tname = getDefaultTableName(context);
+ this.defaultTableName = (tname == null) ? null : new Text(tname);
+
+ if (!simulate) {
+ this.conn = getInstance(context).getConnector(getPrincipal(context), CredentialHelper.extractToken(getTokenClass(context), getToken(context)));
+ mtbw = conn.createMultiTableBatchWriter(getBatchWriterOptions(context));
+ }
+ }
+
+ /**
+ * Push a mutation into a table. If table is null, the defaultTable will be used. If canCreateTable is set, the table will be created if it does not exist.
+ * The table name must only contain alphanumerics and underscore.
+ */
+ @Override
+ public void write(Text table, Mutation mutation) throws IOException {
+ if (table == null || table.toString().isEmpty())
+ table = this.defaultTableName;
+
+ if (!simulate && table == null)
+ throw new IOException("No table or default table specified. Try simulation mode next time");
+
+ ++mutCount;
+ valCount += mutation.size();
+ printMutation(table, mutation);
+
+ if (simulate)
+ return;
+
+ if (!bws.containsKey(table))
+ try {
+ addTable(table);
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new IOException(e);
+ }
+
+ try {
+ bws.get(table).addMutation(mutation);
+ } catch (MutationsRejectedException e) {
+ throw new IOException(e);
+ }
+ }
+
+ public void addTable(Text tableName) throws AccumuloException, AccumuloSecurityException {
+ if (simulate) {
+ log.info("Simulating adding table: " + tableName);
+ return;
+ }
+
+ log.debug("Adding table: " + tableName);
+ BatchWriter bw = null;
+ String table = tableName.toString();
+
+ if (createTables && !conn.tableOperations().exists(table)) {
+ try {
+ conn.tableOperations().create(table);
+ } catch (AccumuloSecurityException e) {
+ log.error("Accumulo security violation creating " + table, e);
+ throw e;
+ } catch (TableExistsException e) {
+ // Shouldn't happen
+ }
+ }
+
+ try {
+ bw = mtbw.getBatchWriter(table);
+ } catch (TableNotFoundException e) {
+ log.error("Accumulo table " + table + " doesn't exist and cannot be created.", e);
+ throw new AccumuloException(e);
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (AccumuloSecurityException e) {
+ throw e;
+ }
+
+ if (bw != null)
+ bws.put(tableName, bw);
+ }
+
+ private int printMutation(Text table, Mutation m) {
+ if (log.isTraceEnabled()) {
+ log.trace(String.format("Table %s row key: %s", table, hexDump(m.getRow())));
+ for (ColumnUpdate cu : m.getUpdates()) {
+ log.trace(String.format("Table %s column: %s:%s", table, hexDump(cu.getColumnFamily()), hexDump(cu.getColumnQualifier())));
+ log.trace(String.format("Table %s security: %s", table, new ColumnVisibility(cu.getColumnVisibility()).toString()));
+ log.trace(String.format("Table %s value: %s", table, hexDump(cu.getValue())));
+ }
+ }
+ return m.getUpdates().size();
+ }
+
+ private String hexDump(byte[] ba) {
+ StringBuilder sb = new StringBuilder();
+ for (byte b : ba) {
+ if ((b > 0x20) && (b < 0x7e))
+ sb.append((char) b);
+ else
+ sb.append(String.format("x%02x", b));
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public void close(TaskAttemptContext attempt) throws IOException, InterruptedException {
+ log.debug("mutations written: " + mutCount + ", values written: " + valCount);
+ if (simulate)
+ return;
+
+ try {
+ mtbw.close();
+ } catch (MutationsRejectedException e) {
+ if (e.getAuthorizationFailuresMap().size() >= 0) {
+ HashMap<String,Set<SecurityErrorCode>> tables = new HashMap<String,Set<SecurityErrorCode>>();
+ for (Entry<KeyExtent,Set<SecurityErrorCode>> ke : e.getAuthorizationFailuresMap().entrySet()) {
+ Set<SecurityErrorCode> secCodes = tables.get(ke.getKey().getTableId().toString());
+ if (secCodes == null) {
+ secCodes = new HashSet<SecurityErrorCode>();
+ tables.put(ke.getKey().getTableId().toString(), secCodes);
+ }
+ secCodes.addAll(ke.getValue());
+ }
+
+ log.error("Not authorized to write to tables : " + tables);
+ }
+
+ if (e.getConstraintViolationSummaries().size() > 0) {
+ log.error("Constraint violations : " + e.getConstraintViolationSummaries().size());
+ }
+ }
+ }
+ }
+
+ @Override
+ public void checkOutputSpecs(JobContext job) throws IOException {
+ if (!isConnectorInfoSet(job))
+ throw new IOException("Connector info has not been set.");
+ try {
+ // if the instance isn't configured, it will complain here
+ Connector c = getInstance(job).getConnector(getPrincipal(job), CredentialHelper.extractToken(getTokenClass(job), getToken(job)));
+ if (!c.securityOperations().authenticateUser(getPrincipal(job), CredentialHelper.extractToken(getTokenClass(job), getToken(job))))
+ throw new IOException("Unable to authenticate user");
+ } catch (AccumuloException e) {
+ throw new IOException(e);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public OutputCommitter getOutputCommitter(TaskAttemptContext context) {
+ return new NullOutputFormat<Text,Mutation>().getOutputCommitter(context);
+ }
+
+ @Override
+ public RecordWriter<Text,Mutation> getRecordWriter(TaskAttemptContext attempt) throws IOException {
+ try {
+ return new AccumuloRecordWriter(attempt);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ // ----------------------------------------------------------------------------------------------------
+ // Everything below this line is deprecated and should go away in future versions
+ // ----------------------------------------------------------------------------------------------------
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setConnectorInfo(Job, String, AuthenticationToken)}, {@link #setCreateTables(Job, boolean)}, and
+ * {@link #setDefaultTableName(Job, String)} instead.
+ */
+ @Deprecated
+ public static void setOutputInfo(Configuration conf, String user, byte[] passwd, boolean createTables, String defaultTable) {
+ try {
+ OutputConfigurator.setConnectorInfo(CLASS, conf, user, new PasswordToken(passwd));
+ } catch (AccumuloSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ OutputConfigurator.setCreateTables(CLASS, conf, createTables);
+ OutputConfigurator.setDefaultTableName(CLASS, conf, defaultTable);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setZooKeeperInstance(Job, String, String)} instead.
+ */
+ @Deprecated
+ public static void setZooKeeperInstance(Configuration conf, String instanceName, String zooKeepers) {
+ OutputConfigurator.setZooKeeperInstance(CLASS, conf, instanceName, zooKeepers);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setMockInstance(Job, String)} instead.
+ */
+ @Deprecated
+ public static void setMockInstance(Configuration conf, String instanceName) {
+ OutputConfigurator.setMockInstance(CLASS, conf, instanceName);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setBatchWriterOptions(Job, BatchWriterConfig)} instead.
+ */
+ @Deprecated
+ public static void setMaxMutationBufferSize(Configuration conf, long numberOfBytes) {
+ BatchWriterConfig bwConfig = OutputConfigurator.getBatchWriterOptions(CLASS, conf);
+ bwConfig.setMaxMemory(numberOfBytes);
+ OutputConfigurator.setBatchWriterOptions(CLASS, conf, bwConfig);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setBatchWriterOptions(Job, BatchWriterConfig)} instead.
+ */
+ @Deprecated
+ public static void setMaxLatency(Configuration conf, int numberOfMilliseconds) {
+ BatchWriterConfig bwConfig = OutputConfigurator.getBatchWriterOptions(CLASS, conf);
+ bwConfig.setMaxLatency(numberOfMilliseconds, TimeUnit.MILLISECONDS);
+ OutputConfigurator.setBatchWriterOptions(CLASS, conf, bwConfig);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setBatchWriterOptions(Job, BatchWriterConfig)} instead.
+ */
+ @Deprecated
+ public static void setMaxWriteThreads(Configuration conf, int numberOfThreads) {
+ BatchWriterConfig bwConfig = OutputConfigurator.getBatchWriterOptions(CLASS, conf);
+ bwConfig.setMaxWriteThreads(numberOfThreads);
+ OutputConfigurator.setBatchWriterOptions(CLASS, conf, bwConfig);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setLogLevel(Job, Level)} instead.
+ */
+ @Deprecated
+ public static void setLogLevel(Configuration conf, Level level) {
+ OutputConfigurator.setLogLevel(CLASS, conf, level);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setSimulationMode(Job, boolean)} instead.
+ */
+ @Deprecated
+ public static void setSimulationMode(Configuration conf) {
+ OutputConfigurator.setSimulationMode(CLASS, conf, true);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getToken(JobContext)} instead.
+ */
+ @Deprecated
+ protected static String getPrincipal(Configuration conf) {
+ return OutputConfigurator.getPrincipal(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getToken(JobContext)} instead.
+ */
+ @Deprecated
+ protected static byte[] getToken(Configuration conf) {
+ return OutputConfigurator.getToken(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #canCreateTables(JobContext)} instead.
+ */
+ @Deprecated
+ protected static boolean canCreateTables(Configuration conf) {
+ return OutputConfigurator.canCreateTables(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getDefaultTableName(JobContext)} instead.
+ */
+ @Deprecated
+ protected static String getDefaultTableName(Configuration conf) {
+ return OutputConfigurator.getDefaultTableName(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getInstance(JobContext)} instead.
+ */
+ @Deprecated
+ protected static Instance getInstance(Configuration conf) {
+ return OutputConfigurator.getInstance(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getBatchWriterOptions(JobContext)} instead.
+ */
+ @Deprecated
+ protected static long getMaxMutationBufferSize(Configuration conf) {
+ return OutputConfigurator.getBatchWriterOptions(CLASS, conf).getMaxMemory();
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getBatchWriterOptions(JobContext)} instead.
+ */
+ @Deprecated
+ protected static int getMaxLatency(Configuration conf) {
+ return (int) OutputConfigurator.getBatchWriterOptions(CLASS, conf).getMaxLatency(TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getBatchWriterOptions(JobContext)} instead.
+ */
+ @Deprecated
+ protected static int getMaxWriteThreads(Configuration conf) {
+ return OutputConfigurator.getBatchWriterOptions(CLASS, conf).getMaxWriteThreads();
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getLogLevel(JobContext)} instead.
+ */
+ @Deprecated
+ protected static Level getLogLevel(Configuration conf) {
+ return OutputConfigurator.getLogLevel(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getSimulationMode(JobContext)} instead.
+ */
+ @Deprecated
+ protected static boolean getSimulationMode(Configuration conf) {
+ return OutputConfigurator.getSimulationMode(CLASS, conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
index 0e9444d,0000000..710c565
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
@@@ -1,1337 -1,0 +1,1336 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Method;
+import java.net.InetAddress;
+import java.net.URLDecoder;
+import java.net.URLEncoder;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.StringTokenizer;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientSideIteratorScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.RowIterator;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableDeletedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.impl.OfflineScanner;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.TabletLocator;
+import org.apache.accumulo.core.client.mapreduce.lib.util.InputConfigurator;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.VersioningIterator;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * This abstract {@link InputFormat} class allows MapReduce jobs to use Accumulo as the source of K,V pairs.
+ * <p>
+ * Subclasses must implement a {@link #createRecordReader(InputSplit, TaskAttemptContext)} to provide a {@link RecordReader} for K,V.
+ * <p>
+ * A static base class, RecordReaderBase, is provided to retrieve Accumulo {@link Key}/{@link Value} pairs, but one must implement its
+ * {@link RecordReaderBase#nextKeyValue()} to transform them to the desired generic types K,V.
+ * <p>
+ * See {@link AccumuloInputFormat} for an example implementation.
+ */
+public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
+
+ private static final Class<?> CLASS = AccumuloInputFormat.class;
+ protected static final Logger log = Logger.getLogger(CLASS);
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
+ * conversion to a string, and is not intended to be secure.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission)
+ * @param token
+ * the user's password
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public static void setConnectorInfo(Job job, String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ InputConfigurator.setConnectorInfo(CLASS, job.getConfiguration(), principal, token);
+ }
+
+ /**
+ * Determines if the connector has been configured.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the connector has been configured, false otherwise
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static Boolean isConnectorInfoSet(JobContext context) {
+ return InputConfigurator.isConnectorInfoSet(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static String getPrincipal(JobContext context) {
+ return InputConfigurator.getPrincipal(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Gets the serialized token class from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static String getTokenClass(JobContext context) {
+ return InputConfigurator.getTokenClass(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Gets the password from the configuration. WARNING: The password is stored in the Configuration and shared with all MapReduce tasks; It is BASE64 encoded to
+ * provide a charset safe conversion to a string, and is not intended to be secure.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the decoded user password
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static byte[] getToken(JobContext context) {
+ return InputConfigurator.getToken(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Configures a {@link ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @param zooKeepers
+ * a comma-separated list of zookeeper servers
+ * @since 1.5.0
+ */
+ public static void setZooKeeperInstance(Job job, String instanceName, String zooKeepers) {
+ InputConfigurator.setZooKeeperInstance(CLASS, job.getConfiguration(), instanceName, zooKeepers);
+ }
+
+ /**
+ * Configures a {@link MockInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @since 1.5.0
+ */
+ public static void setMockInstance(Job job, String instanceName) {
+ InputConfigurator.setMockInstance(CLASS, job.getConfiguration(), instanceName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link Instance} based on the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return an Accumulo instance
+ * @since 1.5.0
+ * @see #setZooKeeperInstance(Job, String, String)
+ * @see #setMockInstance(Job, String)
+ */
+ protected static Instance getInstance(JobContext context) {
+ return InputConfigurator.getInstance(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param level
+ * the logging level
+ * @since 1.5.0
+ */
+ public static void setLogLevel(Job job, Level level) {
+ InputConfigurator.setLogLevel(CLASS, job.getConfiguration(), level);
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the log level
+ * @since 1.5.0
+ * @see #setLogLevel(Job, Level)
+ */
+ protected static Level getLogLevel(JobContext context) {
+ return InputConfigurator.getLogLevel(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Sets the name of the input table, over which this job will scan.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.5.0
+ */
+ public static void setInputTableName(Job job, String tableName) {
+ InputConfigurator.setInputTableName(CLASS, job.getConfiguration(), tableName);
+ }
+
+ /**
+ * Gets the table name from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the table name
+ * @since 1.5.0
+ * @see #setInputTableName(Job, String)
+ */
+ protected static String getInputTableName(JobContext context) {
+ return InputConfigurator.getInputTableName(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Sets the {@link Authorizations} used to scan. Must be a subset of the user's authorization. Defaults to the empty set.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param auths
+ * the user's authorizations
+ * @since 1.5.0
+ */
+ public static void setScanAuthorizations(Job job, Authorizations auths) {
+ InputConfigurator.setScanAuthorizations(CLASS, job.getConfiguration(), auths);
+ }
+
+ /**
+ * Gets the authorizations to set for the scans from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the Accumulo scan authorizations
+ * @since 1.5.0
+ * @see #setScanAuthorizations(Job, Authorizations)
+ */
+ protected static Authorizations getScanAuthorizations(JobContext context) {
+ return InputConfigurator.getScanAuthorizations(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Sets the input ranges to scan for this job. If not set, the entire table will be scanned.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param ranges
+ * the ranges that will be mapped over
+ * @since 1.5.0
+ */
+ public static void setRanges(Job job, Collection<Range> ranges) {
+ InputConfigurator.setRanges(CLASS, job.getConfiguration(), ranges);
+ }
+
+ /**
+ * Gets the ranges to scan over from a job.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the ranges
+ * @throws IOException
+ * if the ranges have been encoded improperly
+ * @since 1.5.0
+ * @see #setRanges(Job, Collection)
+ */
+ protected static List<Range> getRanges(JobContext context) throws IOException {
+ return InputConfigurator.getRanges(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Restricts the columns that will be mapped over for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param columnFamilyColumnQualifierPairs
+ * a pair of {@link Text} objects corresponding to column family and column qualifier. If the column qualifier is null, the entire column family is
+ * selected. An empty set is the default and is equivalent to scanning the all columns.
+ * @since 1.5.0
+ */
+ public static void fetchColumns(Job job, Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
+ InputConfigurator.fetchColumns(CLASS, job.getConfiguration(), columnFamilyColumnQualifierPairs);
+ }
+
+ /**
+ * Gets the columns to be mapped over from this job.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return a set of columns
+ * @since 1.5.0
+ * @see #fetchColumns(Job, Collection)
+ */
+ protected static Set<Pair<Text,Text>> getFetchedColumns(JobContext context) {
+ return InputConfigurator.getFetchedColumns(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Encode an iterator on the input for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param cfg
+ * the configuration of the iterator
+ * @since 1.5.0
+ */
+ public static void addIterator(Job job, IteratorSetting cfg) {
+ InputConfigurator.addIterator(CLASS, job.getConfiguration(), cfg);
+ }
+
+ /**
+ * Gets a list of the iterator settings (for iterators to apply to a scanner) from this configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return a list of iterators
+ * @since 1.5.0
+ * @see #addIterator(Job, IteratorSetting)
+ */
+ protected static List<IteratorSetting> getIterators(JobContext context) {
+ return InputConfigurator.getIterators(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Controls the automatic adjustment of ranges for this job. This feature merges overlapping ranges, then splits them to align with tablet boundaries.
+ * Disabling this feature will cause exactly one Map task to be created for each specified range. The default setting is enabled. *
+ *
+ * <p>
+ * By default, this feature is <b>enabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @see #setRanges(Job, Collection)
+ * @since 1.5.0
+ */
+ public static void setAutoAdjustRanges(Job job, boolean enableFeature) {
+ InputConfigurator.setAutoAdjustRanges(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has auto-adjust ranges enabled.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return false if the feature is disabled, true otherwise
+ * @since 1.5.0
+ * @see #setAutoAdjustRanges(Job, boolean)
+ */
+ protected static boolean getAutoAdjustRanges(JobContext context) {
+ return InputConfigurator.getAutoAdjustRanges(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Controls the use of the {@link IsolatedScanner} in this job.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setScanIsolation(Job job, boolean enableFeature) {
+ InputConfigurator.setScanIsolation(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has isolation enabled.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setScanIsolation(Job, boolean)
+ */
+ protected static boolean isIsolated(JobContext context) {
+ return InputConfigurator.isIsolated(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Controls the use of the {@link ClientSideIteratorScanner} in this job. Enabling this feature will cause the iterator stack to be constructed within the Map
+ * task, rather than within the Accumulo TServer. To use this feature, all classes needed for those iterators must be available on the classpath for the task.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setLocalIterators(Job job, boolean enableFeature) {
+ InputConfigurator.setLocalIterators(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration uses local iterators.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setLocalIterators(Job, boolean)
+ */
+ protected static boolean usesLocalIterators(JobContext context) {
+ return InputConfigurator.usesLocalIterators(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * <p>
+ * Enable reading offline tables. By default, this feature is disabled and only online tables are scanned. This will make the map reduce job directly read the
+ * table's files. If the table is not offline, then the job will fail. If the table comes online during the map reduce job, it is likely that the job will
+ * fail.
+ *
+ * <p>
+ * To use this option, the map reduce user will need access to read the Accumulo directory in HDFS.
+ *
+ * <p>
+ * Reading the offline table will create the scan time iterator stack in the map process. So any iterators that are configured for the table will need to be
+ * on the mapper's classpath.
+ *
+ * <p>
+ * One way to use this feature is to clone a table, take the clone offline, and use the clone as the input table for a map reduce job. If you plan to map
+ * reduce over the data many times, it may be better to the compact the table, clone it, take it offline, and use the clone for all map reduce jobs. The
+ * reason to do this is that compaction will reduce each tablet in the table to one file, and it is faster to read from one file.
+ *
+ * <p>
+ * There are two possible advantages to reading a tables file directly out of HDFS. First, you may see better read performance. Second, it will support
+ * speculative execution better. When reading an online table speculative execution can put more load on an already slow tablet server.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setOfflineTableScan(Job job, boolean enableFeature) {
+ InputConfigurator.setOfflineTableScan(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has the offline table scan feature enabled.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setOfflineTableScan(Job, boolean)
+ */
+ protected static boolean isOfflineScan(JobContext context) {
+ return InputConfigurator.isOfflineScan(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * Initializes an Accumulo {@link TabletLocator} based on the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return an Accumulo tablet locator
+ * @throws TableNotFoundException
+ * if the table name set on the configuration doesn't exist
+ * @since 1.5.0
+ */
+ protected static TabletLocator getTabletLocator(JobContext context) throws TableNotFoundException {
+ return InputConfigurator.getTabletLocator(CLASS, getConfiguration(context));
+ }
+
+ // InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext job)
+ /**
+ * Check whether a configuration is fully configured to be used with an Accumulo {@link org.apache.hadoop.mapreduce.InputFormat}.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @throws IOException
+ * if the context is improperly configured
+ * @since 1.5.0
+ */
+ protected static void validateOptions(JobContext context) throws IOException {
+ InputConfigurator.validateOptions(CLASS, getConfiguration(context));
+ }
+
+ /**
+ * An abstract base class to be used to create {@link RecordReader} instances that convert from Accumulo {@link Key}/{@link Value} pairs to the user's K/V
+ * types.
+ *
+ * Subclasses must implement {@link #nextKeyValue()} and use it to update the following variables:
+ * <ul>
+ * <li>K {@link #currentK}</li>
+ * <li>V {@link #currentV}</li>
+ * <li>Key {@link #currentKey} (used for progress reporting)</li>
+ * <li>int {@link #numKeysRead} (used for progress reporting)</li>
+ * </ul>
+ */
+ protected abstract static class RecordReaderBase<K,V> extends RecordReader<K,V> {
+ protected long numKeysRead;
+ protected Iterator<Entry<Key,Value>> scannerIterator;
+ protected org.apache.accumulo.core.client.mapreduce.RangeInputSplit split;
+
+ /**
+ * Apply the configured iterators from the configuration to the scanner.
+ *
+ * @param scanner
+ * the scanner to configure
+ */
+ protected void setupIterators(List<IteratorSetting> iterators, Scanner scanner) {
+ for (IteratorSetting iterator : iterators) {
+ scanner.addScanIterator(iterator);
+ }
+ }
+
+ /**
+ * Initialize a scanner over the given input split using this task attempt configuration.
+ */
+ @Override
+ public void initialize(InputSplit inSplit, TaskAttemptContext attempt) throws IOException {
+ Scanner scanner;
+ split = (org.apache.accumulo.core.client.mapreduce.RangeInputSplit) inSplit;
+ log.debug("Initializing input split: " + split.getRange());
+
+ Instance instance = split.getInstance();
+ if (null == instance) {
+ instance = getInstance(attempt);
+ }
+
+ String principal = split.getPrincipal();
+ if (null == principal) {
+ principal = getPrincipal(attempt);
+ }
+
+ AuthenticationToken token = split.getToken();
+ if (null == token) {
+ String tokenClass = getTokenClass(attempt);
+ byte[] tokenBytes = getToken(attempt);
+ try {
+ token = CredentialHelper.extractToken(tokenClass, tokenBytes);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+ }
+
+ Authorizations authorizations = split.getAuths();
+ if (null == authorizations) {
+ authorizations = getScanAuthorizations(attempt);
+ }
+
+ String table = split.getTable();
+ if (null == table) {
+ table = getInputTableName(attempt);
+ }
+
+ Boolean isOffline = split.isOffline();
+ if (null == isOffline) {
+ isOffline = isOfflineScan(attempt);
+ }
+
+ Boolean isIsolated = split.isIsolatedScan();
+ if (null == isIsolated) {
+ isIsolated = isIsolated(attempt);
+ }
+
+ Boolean usesLocalIterators = split.usesLocalIterators();
+ if (null == usesLocalIterators) {
+ usesLocalIterators = usesLocalIterators(attempt);
+ }
+
+ List<IteratorSetting> iterators = split.getIterators();
+ if (null == iterators) {
+ iterators = getIterators(attempt);
+ }
+
+ Set<Pair<Text,Text>> columns = split.getFetchedColumns();
+ if (null == columns) {
+ columns = getFetchedColumns(attempt);
+ }
+
+ try {
+ log.debug("Creating connector with user: " + principal);
+ Connector conn = instance.getConnector(principal, token);
+ log.debug("Creating scanner for table: " + table);
+ log.debug("Authorizations are: " + authorizations);
+ if (isOffline) {
+ String tokenClass = token.getClass().getCanonicalName();
+ ByteBuffer tokenBuffer = ByteBuffer.wrap(CredentialHelper.toBytes(token));
+ scanner = new OfflineScanner(instance, new TCredentials(principal, tokenClass, tokenBuffer, instance.getInstanceID()), Tables.getTableId(instance,
+ table), authorizations);
+ } else {
+ scanner = conn.createScanner(table, authorizations);
+ }
+ if (isIsolated) {
+ log.info("Creating isolated scanner");
+ scanner = new IsolatedScanner(scanner);
+ }
+ if (usesLocalIterators) {
+ log.info("Using local iterators");
+ scanner = new ClientSideIteratorScanner(scanner);
+ }
+ setupIterators(iterators, scanner);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ // setup a scanner within the bounds of this split
+ for (Pair<Text,Text> c : columns) {
+ if (c.getSecond() != null) {
+ log.debug("Fetching column " + c.getFirst() + ":" + c.getSecond());
+ scanner.fetchColumn(c.getFirst(), c.getSecond());
+ } else {
+ log.debug("Fetching column family " + c.getFirst());
+ scanner.fetchColumnFamily(c.getFirst());
+ }
+ }
+
+ scanner.setRange(split.getRange());
+
+ numKeysRead = 0;
+
+ // do this last after setting all scanner options
+ scannerIterator = scanner.iterator();
+ }
+
+ @Override
+ public void close() {}
+
+ @Override
+ public float getProgress() throws IOException {
+ if (numKeysRead > 0 && currentKey == null)
+ return 1.0f;
+ return split.getProgress(currentKey);
+ }
+
+ protected K currentK = null;
+ protected V currentV = null;
+ protected Key currentKey = null;
+ protected Value currentValue = null;
+
+ @Override
+ public K getCurrentKey() throws IOException, InterruptedException {
+ return currentK;
+ }
+
+ @Override
+ public V getCurrentValue() throws IOException, InterruptedException {
+ return currentV;
+ }
+ }
+
+ Map<String,Map<KeyExtent,List<Range>>> binOfflineTable(JobContext context, String tableName, List<Range> ranges) throws TableNotFoundException,
+ AccumuloException, AccumuloSecurityException {
+
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+
+ Instance instance = getInstance(context);
+ Connector conn = instance.getConnector(getPrincipal(context), CredentialHelper.extractToken(getTokenClass(context), getToken(context)));
+ String tableId = Tables.getTableId(instance, tableName);
+
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ Tables.clearCache(instance);
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ throw new AccumuloException("Table is online " + tableName + "(" + tableId + ") cannot scan table in offline mode ");
+ }
+ }
+
+ for (Range range : ranges) {
+ Text startRow;
+
+ if (range.getStartKey() != null)
+ startRow = range.getStartKey().getRow();
+ else
+ startRow = new Text();
+
+ Range metadataRange = new Range(new KeyExtent(new Text(tableId), startRow, null).getMetadataEntry(), true, null, false);
+ Scanner scanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ Constants.METADATA_PREV_ROW_COLUMN.fetch(scanner);
+ scanner.fetchColumnFamily(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY);
+ scanner.setRange(metadataRange);
+
+ RowIterator rowIter = new RowIterator(scanner);
+
+ KeyExtent lastExtent = null;
+
+ while (rowIter.hasNext()) {
+ Iterator<Entry<Key,Value>> row = rowIter.next();
+ String last = "";
+ KeyExtent extent = null;
+ String location = null;
+
+ while (row.hasNext()) {
+ Entry<Key,Value> entry = row.next();
+ Key key = entry.getKey();
+
+ if (key.getColumnFamily().equals(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY)) {
+ last = entry.getValue().toString();
+ }
+
+ if (key.getColumnFamily().equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY)
+ || key.getColumnFamily().equals(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY)) {
+ location = entry.getValue().toString();
+ }
+
+ if (Constants.METADATA_PREV_ROW_COLUMN.hasColumns(key)) {
+ extent = new KeyExtent(key.getRow(), entry.getValue());
+ }
+
+ }
+
+ if (location != null)
+ return null;
+
+ if (!extent.getTableId().toString().equals(tableId)) {
+ throw new AccumuloException("Saw unexpected table Id " + tableId + " " + extent);
+ }
+
+ if (lastExtent != null && !extent.isPreviousExtent(lastExtent)) {
+ throw new AccumuloException(" " + lastExtent + " is not previous extent " + extent);
+ }
+
+ Map<KeyExtent,List<Range>> tabletRanges = binnedRanges.get(last);
+ if (tabletRanges == null) {
+ tabletRanges = new HashMap<KeyExtent,List<Range>>();
+ binnedRanges.put(last, tabletRanges);
+ }
+
+ List<Range> rangeList = tabletRanges.get(extent);
+ if (rangeList == null) {
+ rangeList = new ArrayList<Range>();
+ tabletRanges.put(extent, rangeList);
+ }
+
+ rangeList.add(range);
+
+ if (extent.getEndRow() == null || range.afterEndKey(new Key(extent.getEndRow()).followingKey(PartialKey.ROW))) {
+ break;
+ }
+
+ lastExtent = extent;
+ }
+
+ }
+
+ return binnedRanges;
+ }
+
+ /**
+ * Read the metadata table to get tablets and match up ranges to them.
+ */
+ @Override
+ public List<InputSplit> getSplits(JobContext context) throws IOException {
+ Level logLevel = getLogLevel(context);
+ log.setLevel(logLevel);
+
+ validateOptions(context);
+
+ String tableName = getInputTableName(context);
+ boolean autoAdjust = getAutoAdjustRanges(context);
+ List<Range> ranges = autoAdjust ? Range.mergeOverlapping(getRanges(context)) : getRanges(context);
+ Instance instance = getInstance(context);
+ boolean offline = isOfflineScan(context);
+ boolean isolated = isIsolated(context);
+ boolean localIterators = usesLocalIterators(context);
+ boolean mockInstance = (null != instance && MockInstance.class.equals(instance.getClass()));
+ Set<Pair<Text,Text>> fetchedColumns = getFetchedColumns(context);
+ Authorizations auths = getScanAuthorizations(context);
+ String principal = getPrincipal(context);
+ String tokenClass = getTokenClass(context);
+ byte[] tokenBytes = getToken(context);
+
+ AuthenticationToken token;
+ try {
+ token = CredentialHelper.extractToken(tokenClass, tokenBytes);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+
+ List<IteratorSetting> iterators = getIterators(context);
+
+ if (ranges.isEmpty()) {
+ ranges = new ArrayList<Range>(1);
+ ranges.add(new Range());
+ }
+
+ // get the metadata information for these ranges
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+ TabletLocator tl;
+ try {
+ if (isOfflineScan(context)) {
+ binnedRanges = binOfflineTable(context, tableName, ranges);
+ while (binnedRanges == null) {
+ // Some tablets were still online, try again
+ UtilWaitThread.sleep(100 + (int) (Math.random() * 100)); // sleep randomly between 100 and 200 ms
+ binnedRanges = binOfflineTable(context, tableName, ranges);
+ }
+ } else {
+ String tableId = null;
+ tl = getTabletLocator(context);
+ // its possible that the cache could contain complete, but old information about a tables tablets... so clear it
+ tl.invalidateCache();
+ while (!tl.binRanges(ranges, binnedRanges, new TCredentials(principal, tokenClass, ByteBuffer.wrap(tokenBytes), instance.getInstanceID())).isEmpty()) {
+ if (!(instance instanceof MockInstance)) {
+ if (tableId == null)
+ tableId = Tables.getTableId(instance, tableName);
+ if (!Tables.exists(instance, tableId))
+ throw new TableDeletedException(tableId);
+ if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
+ throw new TableOfflineException(instance, tableId);
+ }
+ binnedRanges.clear();
+ log.warn("Unable to locate bins for specified ranges. Retrying.");
+ UtilWaitThread.sleep(100 + (int) (Math.random() * 100)); // sleep randomly between 100 and 200 ms
+ tl.invalidateCache();
+ }
+ }
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ ArrayList<InputSplit> splits = new ArrayList<InputSplit>(ranges.size());
+ HashMap<Range,ArrayList<String>> splitsToAdd = null;
+
+ if (!autoAdjust)
+ splitsToAdd = new HashMap<Range,ArrayList<String>>();
+
+ HashMap<String,String> hostNameCache = new HashMap<String,String>();
+
+ for (Entry<String,Map<KeyExtent,List<Range>>> tserverBin : binnedRanges.entrySet()) {
+ String ip = tserverBin.getKey().split(":", 2)[0];
+ String location = hostNameCache.get(ip);
+ if (location == null) {
+ InetAddress inetAddress = InetAddress.getByName(ip);
+ location = inetAddress.getHostName();
+ hostNameCache.put(ip, location);
+ }
+
+ for (Entry<KeyExtent,List<Range>> extentRanges : tserverBin.getValue().entrySet()) {
+ Range ke = extentRanges.getKey().toDataRange();
+ for (Range r : extentRanges.getValue()) {
+ if (autoAdjust) {
+ // divide ranges into smaller ranges, based on the tablets
+ splits.add(new org.apache.accumulo.core.client.mapreduce.RangeInputSplit(ke.clip(r), new String[] {location}));
+ } else {
+ // don't divide ranges
+ ArrayList<String> locations = splitsToAdd.get(r);
+ if (locations == null)
+ locations = new ArrayList<String>(1);
+ locations.add(location);
+ splitsToAdd.put(r, locations);
+ }
+ }
+ }
+ }
+
+ if (!autoAdjust)
+ for (Entry<Range,ArrayList<String>> entry : splitsToAdd.entrySet())
+ splits.add(new org.apache.accumulo.core.client.mapreduce.RangeInputSplit(entry.getKey(), entry.getValue().toArray(new String[0])));
+
+ for (InputSplit inputSplit : splits) {
+ org.apache.accumulo.core.client.mapreduce.RangeInputSplit split = (org.apache.accumulo.core.client.mapreduce.RangeInputSplit) inputSplit;
+
+ split.setTable(tableName);
+ split.setOffline(offline);
+ split.setIsolatedScan(isolated);
+ split.setUsesLocalIterators(localIterators);
+ split.setMockInstance(mockInstance);
+ split.setFetchedColumns(fetchedColumns);
+ split.setPrincipal(principal);
+ split.setToken(token);
+ split.setInstanceName(instance.getInstanceName());
+ split.setZooKeepers(instance.getZooKeepers());
+ split.setAuths(auths);
+ split.setIterators(iterators);
+ split.setLogLevel(logLevel);
+ }
+
+ return splits;
+ }
+
+ // ----------------------------------------------------------------------------------------------------
+ // Everything below this line is deprecated and should go away in future versions
+ // ----------------------------------------------------------------------------------------------------
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setScanIsolation(Job, boolean)} instead.
+ */
+ @Deprecated
+ public static void setIsolated(Configuration conf, boolean enable) {
+ InputConfigurator.setScanIsolation(CLASS, conf, enable);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setLocalIterators(Job, boolean)} instead.
+ */
+ @Deprecated
+ public static void setLocalIterators(Configuration conf, boolean enable) {
+ InputConfigurator.setLocalIterators(CLASS, conf, enable);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setConnectorInfo(Job, String, AuthenticationToken)}, {@link #setInputTableName(Job, String)}, and
+ * {@link #setScanAuthorizations(Job, Authorizations)} instead.
+ */
+ @Deprecated
+ public static void setInputInfo(Configuration conf, String user, byte[] passwd, String table, Authorizations auths) {
+ try {
+ InputConfigurator.setConnectorInfo(CLASS, conf, user, new PasswordToken(passwd));
+ } catch (AccumuloSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ InputConfigurator.setInputTableName(CLASS, conf, table);
+ InputConfigurator.setScanAuthorizations(CLASS, conf, auths);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setZooKeeperInstance(Job, String, String)} instead.
+ */
+ @Deprecated
+ public static void setZooKeeperInstance(Configuration conf, String instanceName, String zooKeepers) {
+ InputConfigurator.setZooKeeperInstance(CLASS, conf, instanceName, zooKeepers);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setMockInstance(Job, String)} instead.
+ */
+ @Deprecated
+ public static void setMockInstance(Configuration conf, String instanceName) {
+ InputConfigurator.setMockInstance(CLASS, conf, instanceName);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setRanges(Job, Collection)} instead.
+ */
+ @Deprecated
+ public static void setRanges(Configuration conf, Collection<Range> ranges) {
+ InputConfigurator.setRanges(CLASS, conf, ranges);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setAutoAdjustRanges(Job, boolean)} instead.
+ */
+ @Deprecated
+ public static void disableAutoAdjustRanges(Configuration conf) {
+ InputConfigurator.setAutoAdjustRanges(CLASS, conf, false);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #addIterator(Job, IteratorSetting)} to add the {@link VersioningIterator} instead.
+ */
+ @Deprecated
+ public static void setMaxVersions(Configuration conf, int maxVersions) throws IOException {
+ IteratorSetting vers = new IteratorSetting(1, "vers", VersioningIterator.class);
+ try {
+ VersioningIterator.setMaxVersions(vers, maxVersions);
+ } catch (IllegalArgumentException e) {
+ throw new IOException(e);
+ }
+ InputConfigurator.addIterator(CLASS, conf, vers);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setOfflineTableScan(Job, boolean)} instead.
+ */
+ @Deprecated
+ public static void setScanOffline(Configuration conf, boolean scanOff) {
+ InputConfigurator.setOfflineTableScan(CLASS, conf, scanOff);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #fetchColumns(Job, Collection)} instead.
+ */
+ @Deprecated
+ public static void fetchColumns(Configuration conf, Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
+ InputConfigurator.fetchColumns(CLASS, conf, columnFamilyColumnQualifierPairs);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #setLogLevel(Job, Level)} instead.
+ */
+ @Deprecated
+ public static void setLogLevel(Configuration conf, Level level) {
+ InputConfigurator.setLogLevel(CLASS, conf, level);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #addIterator(Job, IteratorSetting)} instead.
+ */
+ @Deprecated
+ public static void addIterator(Configuration conf, IteratorSetting cfg) {
+ InputConfigurator.addIterator(CLASS, conf, cfg);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #isIsolated(JobContext)} instead.
+ */
+ @Deprecated
+ protected static boolean isIsolated(Configuration conf) {
+ return InputConfigurator.isIsolated(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #usesLocalIterators(JobContext)} instead.
+ */
+ @Deprecated
+ protected static boolean usesLocalIterators(Configuration conf) {
+ return InputConfigurator.usesLocalIterators(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getPrincipal(JobContext)} instead.
+ */
+ @Deprecated
+ protected static String getPrincipal(Configuration conf) {
+ return InputConfigurator.getPrincipal(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getToken(JobContext)} instead.
+ */
+ @Deprecated
+ protected static byte[] getToken(Configuration conf) {
+ return InputConfigurator.getToken(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getInputTableName(JobContext)} instead.
+ */
+ @Deprecated
+ protected static String getTablename(Configuration conf) {
+ return InputConfigurator.getInputTableName(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getScanAuthorizations(JobContext)} instead.
+ */
+ @Deprecated
+ protected static Authorizations getAuthorizations(Configuration conf) {
+ return InputConfigurator.getScanAuthorizations(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getInstance(JobContext)} instead.
+ */
+ @Deprecated
+ protected static Instance getInstance(Configuration conf) {
+ return InputConfigurator.getInstance(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getTabletLocator(JobContext)} instead.
+ */
+ @Deprecated
+ protected static TabletLocator getTabletLocator(Configuration conf) throws TableNotFoundException {
+ return InputConfigurator.getTabletLocator(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getRanges(JobContext)} instead.
+ */
+ @Deprecated
+ protected static List<Range> getRanges(Configuration conf) throws IOException {
+ return InputConfigurator.getRanges(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getFetchedColumns(JobContext)} instead.
+ */
+ @Deprecated
+ protected static Set<Pair<Text,Text>> getFetchedColumns(Configuration conf) {
+ return InputConfigurator.getFetchedColumns(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getAutoAdjustRanges(JobContext)} instead.
+ */
+ @Deprecated
+ protected static boolean getAutoAdjustRanges(Configuration conf) {
+ return InputConfigurator.getAutoAdjustRanges(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getLogLevel(JobContext)} instead.
+ */
+ @Deprecated
+ protected static Level getLogLevel(Configuration conf) {
+ return InputConfigurator.getLogLevel(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #validateOptions(JobContext)} instead.
+ */
+ @Deprecated
+ protected static void validateOptions(Configuration conf) throws IOException {
+ InputConfigurator.validateOptions(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #addIterator(Job, IteratorSetting)} to add the {@link VersioningIterator} instead.
+ */
+ @Deprecated
+ protected static int getMaxVersions(Configuration conf) {
+ // This is so convoluted, because the only reason to get the number of maxVersions is to construct the same type of IteratorSetting object we have to
+ // deconstruct to get at this option in the first place, but to preserve correct behavior, this appears necessary.
+ List<IteratorSetting> iteratorSettings = InputConfigurator.getIterators(CLASS, conf);
+ for (IteratorSetting setting : iteratorSettings) {
+ if ("vers".equals(setting.getName()) && 1 == setting.getPriority() && VersioningIterator.class.getName().equals(setting.getIteratorClass())) {
+ if (setting.getOptions().containsKey("maxVersions"))
+ return Integer.parseInt(setting.getOptions().get("maxVersions"));
+ else
+ return -1;
+ }
+ }
+ return -1;
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #isOfflineScan(JobContext)} instead.
+ */
+ @Deprecated
+ protected static boolean isOfflineScan(Configuration conf) {
+ return InputConfigurator.isOfflineScan(CLASS, conf);
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getIterators(JobContext)} instead.
+ */
+ @Deprecated
+ protected static List<AccumuloIterator> getIterators(Configuration conf) {
+ List<IteratorSetting> iteratorSettings = InputConfigurator.getIterators(CLASS, conf);
+ List<AccumuloIterator> deprecatedIterators = new ArrayList<AccumuloIterator>(iteratorSettings.size());
+ for (IteratorSetting setting : iteratorSettings) {
+ AccumuloIterator deprecatedIter = new AccumuloIterator(setting.getPriority() + AccumuloIterator.FIELD_SEP + setting.getIteratorClass()
+ + AccumuloIterator.FIELD_SEP + setting.getName());
+ deprecatedIterators.add(deprecatedIter);
+ }
+ return deprecatedIterators;
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link #getIterators(JobContext)} instead.
+ */
+ @Deprecated
+ protected static List<AccumuloIteratorOption> getIteratorOptions(Configuration conf) {
+ List<IteratorSetting> iteratorSettings = InputConfigurator.getIterators(CLASS, conf);
+ List<AccumuloIteratorOption> deprecatedIteratorOptions = new ArrayList<AccumuloIteratorOption>(iteratorSettings.size());
+ for (IteratorSetting setting : iteratorSettings) {
+ for (Entry<String,String> opt : setting.getOptions().entrySet()) {
+ String deprecatedOption;
+ try {
+ deprecatedOption = setting.getName() + AccumuloIteratorOption.FIELD_SEP + URLEncoder.encode(opt.getKey(), "UTF-8") + AccumuloIteratorOption.FIELD_SEP
+ + URLEncoder.encode(opt.getValue(), "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ deprecatedIteratorOptions.add(new AccumuloIteratorOption(deprecatedOption));
+ }
+ }
+ return deprecatedIteratorOptions;
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link IteratorSetting} instead.
+ */
+ @Deprecated
+ static class AccumuloIterator {
+
+ private static final String FIELD_SEP = ":";
+
+ private int priority;
+ private String iteratorClass;
+ private String iteratorName;
+
+ public AccumuloIterator(int priority, String iteratorClass, String iteratorName) {
+ this.priority = priority;
+ this.iteratorClass = iteratorClass;
+ this.iteratorName = iteratorName;
+ }
+
+ // Parses out a setting given an string supplied from an earlier toString() call
+ public AccumuloIterator(String iteratorSetting) {
+ // Parse the string to expand the iterator
+ StringTokenizer tokenizer = new StringTokenizer(iteratorSetting, FIELD_SEP);
+ priority = Integer.parseInt(tokenizer.nextToken());
+ iteratorClass = tokenizer.nextToken();
+ iteratorName = tokenizer.nextToken();
+ }
+
+ public int getPriority() {
+ return priority;
+ }
+
+ public String getIteratorClass() {
+ return iteratorClass;
+ }
+
+ public String getIteratorName() {
+ return iteratorName;
+ }
+
+ @Override
+ public String toString() {
+ return priority + FIELD_SEP + iteratorClass + FIELD_SEP + iteratorName;
+ }
+
+ }
+
+ /**
+ * @deprecated since 1.5.0; Use {@link IteratorSetting} instead.
+ */
+ @Deprecated
+ static class AccumuloIteratorOption {
+ private static final String FIELD_SEP = ":";
+
+ private String iteratorName;
+ private String key;
+ private String value;
+
+ public AccumuloIteratorOption(String iteratorName, String key, String value) {
+ this.iteratorName = iteratorName;
+ this.key = key;
+ this.value = value;
+ }
+
+ // Parses out an option given a string supplied from an earlier toString() call
+ public AccumuloIteratorOption(String iteratorOption) {
+ StringTokenizer tokenizer = new StringTokenizer(iteratorOption, FIELD_SEP);
+ this.iteratorName = tokenizer.nextToken();
+ try {
+ this.key = URLDecoder.decode(tokenizer.nextToken(), "UTF-8");
+ this.value = URLDecoder.decode(tokenizer.nextToken(), "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public String getIteratorName() {
+ return iteratorName;
+ }
+
+ public String getKey() {
+ return key;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return iteratorName + FIELD_SEP + URLEncoder.encode(key, "UTF-8") + FIELD_SEP + URLEncoder.encode(value, "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+ // use reflection to pull the Configuration out of the JobContext for Hadoop 1 and Hadoop 2 compatibility
+ static Configuration getConfiguration(JobContext context) {
+ try {
+ Class<?> c = InputFormatBase.class.getClassLoader().loadClass("org.apache.hadoop.mapreduce.JobContext");
+ Method m = c.getMethod("getConfiguration");
+ Object o = m.invoke(context, new Object[0]);
+ return (Configuration) o;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * @deprecated since 1.5.2; Use {@link org.apache.accumulo.core.client.mapreduce.RangeInputSplit} instead.
+ * @see org.apache.accumulo.core.client.mapreduce.RangeInputSplit
+ */
+ @Deprecated
+ public static class RangeInputSplit extends org.apache.accumulo.core.client.mapreduce.RangeInputSplit {
+
+ public RangeInputSplit() {
+ super();
+ }
+
+ public RangeInputSplit(Range range, String[] locations) {
+ super(range, locations);
+ }
+ }
+}
[41/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
index f4408f5,0000000..ae5e395
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
@@@ -1,462 -1,0 +1,456 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Level;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class AccumuloInputFormatTest {
+
+ private static final String PREFIX = AccumuloInputFormatTest.class.getSimpleName();
+
+ /**
+ * Test basic setting & getting of max versions.
+ *
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @Deprecated
+ @Test
+ public void testMaxVersions() throws IOException {
+ Job job = new Job();
+ AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 1);
+ int version = AccumuloInputFormat.getMaxVersions(job.getConfiguration());
+ assertEquals(1, version);
+ }
+
+ /**
+ * Test max versions with an invalid value.
+ *
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @Deprecated
+ @Test(expected = IOException.class)
+ public void testMaxVersionsLessThan1() throws IOException {
+ Job job = new Job();
+ AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 0);
+ }
+
+ /**
+ * Test no max version configured.
- *
- * @throws IOException
+ */
+ @Deprecated
+ @Test
+ public void testNoMaxVersion() throws IOException {
+ Job job = new Job();
+ assertEquals(-1, AccumuloInputFormat.getMaxVersions(job.getConfiguration()));
+ }
+
+ /**
+ * Check that the iterator configuration is getting stored in the Job conf correctly.
- *
- * @throws IOException
+ */
+ @Test
+ public void testSetIterator() throws IOException {
+ Job job = new Job();
+
+ IteratorSetting is = new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator");
+ AccumuloInputFormat.addIterator(job, is);
+ Configuration conf = job.getConfiguration();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ is.write(new DataOutputStream(baos));
+ String iterators = conf.get("AccumuloInputFormat.ScanOpts.Iterators");
+ assertEquals(new String(Base64.encodeBase64(baos.toByteArray())), iterators);
+ }
+
+ @Test
+ public void testAddIterator() throws IOException {
+ Job job = new Job();
+
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", WholeRowIterator.class));
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
+ IteratorSetting iter = new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator");
+ iter.addOption("v1", "1");
+ iter.addOption("junk", "\0omg:!\\xyzzy");
+ AccumuloInputFormat.addIterator(job, iter);
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+
+ // Check the list size
+ assertTrue(list.size() == 3);
+
+ // Walk the list and make sure our settings are correct
+ IteratorSetting setting = list.get(0);
+ assertEquals(1, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.user.WholeRowIterator", setting.getIteratorClass());
+ assertEquals("WholeRow", setting.getName());
+ assertEquals(0, setting.getOptions().size());
+
+ setting = list.get(1);
+ assertEquals(2, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
+ assertEquals("Versions", setting.getName());
+ assertEquals(0, setting.getOptions().size());
+
+ setting = list.get(2);
+ assertEquals(3, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
+ assertEquals("Count", setting.getName());
+ assertEquals(2, setting.getOptions().size());
+ assertEquals("1", setting.getOptions().get("v1"));
+ assertEquals("\0omg:!\\xyzzy", setting.getOptions().get("junk"));
+ }
+
+ /**
+ * Test adding iterator options where the keys and values contain both the FIELD_SEPARATOR character (':') and ITERATOR_SEPARATOR (',') characters. There
+ * should be no exceptions thrown when trying to parse these types of option entries.
+ *
+ * This test makes sure that the expected raw values, as appears in the Job, are equal to what's expected.
+ */
+ @Test
+ public void testIteratorOptionEncoding() throws Throwable {
+ String key = "colon:delimited:key";
+ String value = "comma,delimited,value";
+ IteratorSetting someSetting = new IteratorSetting(1, "iterator", "Iterator.class");
+ someSetting.addOption(key, value);
+ Job job = new Job();
+ AccumuloInputFormat.addIterator(job, someSetting);
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+ assertEquals(1, list.size());
+ assertEquals(1, list.get(0).getOptions().size());
+ assertEquals(list.get(0).getOptions().get(key), value);
+
+ someSetting.addOption(key + "2", value);
+ someSetting.setPriority(2);
+ someSetting.setName("it2");
+ AccumuloInputFormat.addIterator(job, someSetting);
+ list = AccumuloInputFormat.getIterators(job);
+ assertEquals(2, list.size());
+ assertEquals(1, list.get(0).getOptions().size());
+ assertEquals(list.get(0).getOptions().get(key), value);
+ assertEquals(2, list.get(1).getOptions().size());
+ assertEquals(list.get(1).getOptions().get(key), value);
+ assertEquals(list.get(1).getOptions().get(key + "2"), value);
+ }
+
+ /**
+ * Test getting iterator settings for multiple iterators set
- *
- * @throws IOException
+ */
+ @Test
+ public void testGetIteratorSettings() throws IOException {
+ Job job = new Job();
+
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator"));
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator"));
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+
+ // Check the list size
+ assertTrue(list.size() == 3);
+
+ // Walk the list and make sure our settings are correct
+ IteratorSetting setting = list.get(0);
+ assertEquals(1, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.WholeRowIterator", setting.getIteratorClass());
+ assertEquals("WholeRow", setting.getName());
+
+ setting = list.get(1);
+ assertEquals(2, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
+ assertEquals("Versions", setting.getName());
+
+ setting = list.get(2);
+ assertEquals(3, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
+ assertEquals("Count", setting.getName());
+
+ }
+
+ @Test
+ public void testSetRegex() throws IOException {
+ Job job = new Job();
+
+ String regex = ">\"*%<>\'\\";
+
+ IteratorSetting is = new IteratorSetting(50, regex, RegExFilter.class);
+ RegExFilter.setRegexs(is, regex, null, null, null, false);
+ AccumuloInputFormat.addIterator(job, is);
+
+ assertTrue(regex.equals(AccumuloInputFormat.getIterators(job).get(0).getName()));
+ }
+
+ private static AssertionError e1 = null;
+ private static AssertionError e2 = null;
+
+ private static class MRTester extends Configured implements Tool {
+ private static class TestMapper extends Mapper<Key,Value,Key,Value> {
+ Key key = null;
+ int count = 0;
+
+ @Override
+ protected void map(Key k, Value v, Context context) throws IOException, InterruptedException {
+ try {
+ if (key != null)
+ assertEquals(key.getRow().toString(), new String(v.get()));
+ assertEquals(k.getRow(), new Text(String.format("%09x", count + 1)));
+ assertEquals(new String(v.get()), String.format("%09x", count));
+ } catch (AssertionError e) {
+ e1 = e;
+ }
+ key = new Key(k);
+ count++;
+ }
+
+ @Override
+ protected void cleanup(Context context) throws IOException, InterruptedException {
+ try {
+ assertEquals(100, count);
+ } catch (AssertionError e) {
+ e2 = e;
+ }
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+
+ if (args.length != 5) {
+ throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <user> <pass> <table> <instanceName> <inputFormatClass>");
+ }
+
+ String user = args[0];
+ String pass = args[1];
+ String table = args[2];
+ String instanceName = args[3];
+ String inputFormatClassName = args[4];
+ @SuppressWarnings("unchecked")
+ Class<? extends InputFormat<?,?>> inputFormatClass = (Class<? extends InputFormat<?,?>>) Class.forName(inputFormatClassName);
+
+ Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+ job.setJarByClass(this.getClass());
+
+ job.setInputFormatClass(inputFormatClass);
+
+ AccumuloInputFormat.setConnectorInfo(job, user, new PasswordToken(pass));
+ AccumuloInputFormat.setInputTableName(job, table);
+ AccumuloInputFormat.setMockInstance(job, instanceName);
+
+ job.setMapperClass(TestMapper.class);
+ job.setMapOutputKeyClass(Key.class);
+ job.setMapOutputValueClass(Value.class);
+ job.setOutputFormatClass(NullOutputFormat.class);
+
+ job.setNumReduceTasks(0);
+
+ job.waitForCompletion(true);
+
+ return job.isSuccessful() ? 0 : 1;
+ }
+
+ public static int main(String[] args) throws Exception {
+ return ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args);
+ }
+ }
+
+ @Test
+ public void testMap() throws Exception {
+ final String INSTANCE_NAME = PREFIX + "_mapreduce_instance";
+ final String TEST_TABLE_1 = PREFIX + "_mapreduce_table_1";
+
+ MockInstance mockInstance = new MockInstance(INSTANCE_NAME);
+ Connector c = mockInstance.getConnector("root", new PasswordToken(""));
+ c.tableOperations().create(TEST_TABLE_1);
+ BatchWriter bw = c.createBatchWriter(TEST_TABLE_1, new BatchWriterConfig());
+ for (int i = 0; i < 100; i++) {
+ Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
+ m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
+ bw.addMutation(m);
+ }
+ bw.close();
+
+ Assert.assertEquals(0, MRTester.main(new String[] {"root", "", TEST_TABLE_1, INSTANCE_NAME, AccumuloInputFormat.class.getCanonicalName()}));
+ assertNull(e1);
+ assertNull(e2);
+ }
+
+ @Test
+ public void testCorrectRangeInputSplits() throws Exception {
+ Job job = new Job(new Configuration(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+
+ String username = "user", table = "table", instance = "instance";
+ PasswordToken password = new PasswordToken("password");
+ Authorizations auths = new Authorizations("foo");
+ Collection<Pair<Text,Text>> fetchColumns = Collections.singleton(new Pair<Text,Text>(new Text("foo"), new Text("bar")));
+ boolean isolated = true, localIters = true;
+ Level level = Level.WARN;
+
+ Instance inst = new MockInstance(instance);
+ Connector connector = inst.getConnector(username, password);
+ connector.tableOperations().create(table);
+
+ AccumuloInputFormat.setConnectorInfo(job, username, password);
+ AccumuloInputFormat.setInputTableName(job, table);
+ AccumuloInputFormat.setScanAuthorizations(job, auths);
+ AccumuloInputFormat.setMockInstance(job, instance);
+ AccumuloInputFormat.setScanIsolation(job, isolated);
+ AccumuloInputFormat.setLocalIterators(job, localIters);
+ AccumuloInputFormat.fetchColumns(job, fetchColumns);
+ AccumuloInputFormat.setLogLevel(job, level);
+
+ AccumuloInputFormat aif = new AccumuloInputFormat();
+
+ List<InputSplit> splits = aif.getSplits(job);
+
+ Assert.assertEquals(1, splits.size());
+
+ InputSplit split = splits.get(0);
+
+ Assert.assertEquals(RangeInputSplit.class, split.getClass());
+
+ RangeInputSplit risplit = (RangeInputSplit) split;
+
+ Assert.assertEquals(username, risplit.getPrincipal());
+ Assert.assertEquals(table, risplit.getTable());
+ Assert.assertEquals(password, risplit.getToken());
+ Assert.assertEquals(auths, risplit.getAuths());
+ Assert.assertEquals(instance, risplit.getInstanceName());
+ Assert.assertEquals(isolated, risplit.isIsolatedScan());
+ Assert.assertEquals(localIters, risplit.usesLocalIterators());
+ Assert.assertEquals(fetchColumns, risplit.getFetchedColumns());
+ Assert.assertEquals(level, risplit.getLogLevel());
+ }
+
+ static class TestMapper extends Mapper<Key,Value,Key,Value> {
+ Key key = null;
+ int count = 0;
+
+ @Override
+ protected void map(Key k, Value v, Context context) throws IOException, InterruptedException {
+ if (key != null)
+ assertEquals(key.getRow().toString(), new String(v.get()));
+ assertEquals(k.getRow(), new Text(String.format("%09x", count + 1)));
+ assertEquals(new String(v.get()), String.format("%09x", count));
+ key = new Key(k);
+ count++;
+ }
+ }
+
+ @Test
+ public void testPartialInputSplitDelegationToConfiguration() throws Exception {
+ String user = "testPartialInputSplitUser";
+ PasswordToken password = new PasswordToken("");
+
+ MockInstance mockInstance = new MockInstance("testPartialInputSplitDelegationToConfiguration");
+ Connector c = mockInstance.getConnector(user, password);
+ c.tableOperations().create("testtable");
+ BatchWriter bw = c.createBatchWriter("testtable", new BatchWriterConfig());
+ for (int i = 0; i < 100; i++) {
+ Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
+ m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
+ bw.addMutation(m);
+ }
+ bw.close();
+
+ Assert.assertEquals(
+ 0,
+ MRTester.main(new String[] {user, "", "testtable", "testPartialInputSplitDelegationToConfiguration",
+ EmptySplitsAccumuloInputFormat.class.getCanonicalName()}));
+ assertNull(e1);
+ assertNull(e2);
+ }
+
+ @Test
+ public void testPartialFailedInputSplitDelegationToConfiguration() throws Exception {
+ String user = "testPartialFailedInputSplit";
+ PasswordToken password = new PasswordToken("");
+
+ MockInstance mockInstance = new MockInstance("testPartialFailedInputSplitDelegationToConfiguration");
+ Connector c = mockInstance.getConnector(user, password);
+ c.tableOperations().create("testtable");
+ BatchWriter bw = c.createBatchWriter("testtable", new BatchWriterConfig());
+ for (int i = 0; i < 100; i++) {
+ Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
+ m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
+ bw.addMutation(m);
+ }
+ bw.close();
+
+ // We should fail before we even get into the Mapper because we can't make the RecordReader
+ Assert.assertEquals(
+ 1,
+ MRTester.main(new String[] {user, "", "testtable", "testPartialFailedInputSplitDelegationToConfiguration",
+ BadPasswordSplitsAccumuloInputFormat.class.getCanonicalName()}));
+ assertNull(e1);
+ assertNull(e2);
+ }
+
+ @Test
+ public void testEmptyColumnFamily() throws IOException {
+ Job job = new Job();
+ Set<Pair<Text,Text>> cols = new HashSet<Pair<Text,Text>>();
+ cols.add(new Pair<Text,Text>(new Text(""), null));
+ cols.add(new Pair<Text,Text>(new Text("foo"), new Text("bar")));
+ cols.add(new Pair<Text,Text>(new Text(""), new Text("bar")));
+ cols.add(new Pair<Text,Text>(new Text(""), new Text("")));
+ cols.add(new Pair<Text,Text>(new Text("foo"), new Text("")));
+ AccumuloInputFormat.fetchColumns(job, cols);
+ Set<Pair<Text,Text>> setCols = AccumuloInputFormat.getFetchedColumns(job);
+ assertEquals(cols, setCols);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/test/java/org/apache/accumulo/core/util/shell/command/FormatterCommandTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/util/shell/command/FormatterCommandTest.java
index 0eb2653,0000000..6000817
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/util/shell/command/FormatterCommandTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/shell/command/FormatterCommandTest.java
@@@ -1,212 -1,0 +1,199 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.util.shell.command;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.util.Iterator;
+import java.util.Map.Entry;
+
- import org.junit.Assert;
-
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.mock.MockShell;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.format.Formatter;
+import org.apache.accumulo.core.util.shell.Shell;
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
++import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Uses the MockShell to test the shell output with Formatters
+ */
+public class FormatterCommandTest {
+ Writer writer = null;
+ InputStream in = null;
+
+ @Test
+ public void test() throws IOException, AccumuloException, AccumuloSecurityException, TableExistsException, ClassNotFoundException {
+ // Keep the Shell AUDIT log off the test output
+ Logger.getLogger(Shell.class).setLevel(Level.WARN);
+
+ final String[] args = new String[] {"--fake", "-u", "root", "-p", ""};
+
+ final String[] commands = createCommands();
+
+ in = MockShell.makeCommands(commands);
+ writer = new StringWriter();
+
+ final MockShell shell = new MockShell(in, writer);
+ shell.config(args);
+
+ // Can't call createtable in the shell with MockAccumulo
+ shell.getConnector().tableOperations().create("test");
+
+ try {
+ shell.start();
+ } catch (Exception e) {
+ Assert.fail("Exception while running commands: " + e.getMessage());
+ }
+
+ shell.getReader().flushConsole();
+
+ final String[] output = StringUtils.split(writer.toString(), '\n');
+
+ boolean formatterOn = false;
+
+ final String[] expectedDefault = new String[] {
+ "row cf:cq [] 1234abcd",
+ "row cf1:cq1 [] 9876fedc",
+ "row2 cf:cq [] 13579bdf",
+ "row2 cf1:cq [] 2468ace"
+ };
+
+ final String[] expectedFormatted = new String[] {
+ "row cf:cq [] 0x31 0x32 0x33 0x34 0x61 0x62 0x63 0x64",
+ "row cf1:cq1 [] 0x39 0x38 0x37 0x36 0x66 0x65 0x64 0x63",
+ "row2 cf:cq [] 0x31 0x33 0x35 0x37 0x39 0x62 0x64 0x66",
+ "row2 cf1:cq [] 0x32 0x34 0x36 0x38 0x61 0x63 0x65"
+ };
+
+ int outputIndex = 0;
+ while (outputIndex < output.length) {
+ final String line = output[outputIndex];
+
+ if (line.startsWith("root@mock-instance")) {
+ if (line.contains("formatter -t test -f org.apache.accumulo.core.util.shell.command.FormatterCommandTest$HexFormatter")) {
+ formatterOn = true;
+ }
+
+ outputIndex++;
+ } else if (line.startsWith("row")) {
+ int expectedIndex = 0;
+ String[] comparisonData;
+
+ // Pick the type of data we expect (formatted or default)
+ if (formatterOn) {
+ comparisonData = expectedFormatted;
+ } else {
+ comparisonData = expectedDefault;
+ }
+
+ // Ensure each output is what we expected
+ while (expectedIndex + outputIndex < output.length && expectedIndex < expectedFormatted.length) {
+ Assert.assertEquals(comparisonData[expectedIndex].trim(), output[expectedIndex + outputIndex].trim());
+ expectedIndex++;
+ }
+
+ outputIndex += expectedIndex;
+ }
+ }
+ }
+
+ private String[] createCommands() {
+ return new String[] {
+ "table test",
+ "insert row cf cq 1234abcd",
+ "insert row cf1 cq1 9876fedc",
+ "insert row2 cf cq 13579bdf",
+ "insert row2 cf1 cq 2468ace",
+ "scan",
+ "formatter -t test -f org.apache.accumulo.core.util.shell.command.FormatterCommandTest$HexFormatter",
+ "scan"
+ };
+ }
+
+ /**
+ * <p>Simple <code>Formatter</code> that will convert each character in the Value
+ * from decimal to hexadecimal. Will automatically skip over characters in the value
+ * which do not fall within the [0-9,a-f] range.</p>
+ *
+ * <p>Example: <code>'0'</code> will be displayed as <code>'0x30'</code></p>
+ */
+ public static class HexFormatter implements Formatter {
+ private Iterator<Entry<Key, Value>> iter = null;
+ private boolean printTs = false;
+
+ private final static String tab = "\t";
+ private final static String newline = "\n";
+
+ public HexFormatter() {}
+
- /* (non-Javadoc)
- * @see java.util.Iterator#hasNext()
- */
+ @Override
+ public boolean hasNext() {
+ return this.iter.hasNext();
+ }
+
- /* (non-Javadoc)
- * @see java.util.Iterator#next()
- */
+ @Override
+ public String next() {
+ final Entry<Key, Value> entry = iter.next();
+
+ String key;
+
+ // Observe the timestamps
+ if (printTs) {
+ key = entry.getKey().toString();
+ } else {
+ key = entry.getKey().toStringNoTime();
+ }
+
+ final Value v = entry.getValue();
+
+ // Approximate how much space we'll need
+ final StringBuilder sb = new StringBuilder(key.length() + v.getSize() * 5);
+
+ sb.append(key).append(tab);
+
+ for (byte b : v.get()) {
+ if ((b >= 48 && b <= 57) || (b >= 97 || b <= 102)) {
+ sb.append(String.format("0x%x ", Integer.valueOf(b)));
+ }
+ }
+
+ sb.append(newline);
+
+ return sb.toString();
+ }
+
- /* (non-Javadoc)
- * @see java.util.Iterator#remove()
- */
+ @Override
+ public void remove() {
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.util.format.Formatter#initialize(java.lang.Iterable, boolean)
- */
+ @Override
+ public void initialize(final Iterable<Entry<Key,Value>> scanner, final boolean printTimestamps) {
+ this.iter = scanner.iterator();
+ this.printTs = printTimestamps;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
index 2ae82b4,0000000..a3bcf62
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
@@@ -1,234 -1,0 +1,229 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.client;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+import java.util.Random;
+
+import org.apache.accumulo.core.cli.BatchScannerOpts;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Internal class used to verify validity of data read.
+ */
+class CountingVerifyingReceiver {
+ private static final Logger log = Logger.getLogger(CountingVerifyingReceiver.class);
+
+ long count = 0;
+ int expectedValueSize = 0;
+ HashMap<Text,Boolean> expectedRows;
+
+ CountingVerifyingReceiver(HashMap<Text,Boolean> expectedRows, int expectedValueSize) {
+ this.expectedRows = expectedRows;
+ this.expectedValueSize = expectedValueSize;
+ }
+
+ public void receive(Key key, Value value) {
+
+ String row = key.getRow().toString();
+ long rowid = Integer.parseInt(row.split("_")[1]);
+
+ byte expectedValue[] = RandomBatchWriter.createValue(rowid, expectedValueSize);
+
+ if (!Arrays.equals(expectedValue, value.get())) {
+ log.error("Got unexpected value for " + key + " expected : " + new String(expectedValue) + " got : " + new String(value.get()));
+ }
+
+ if (!expectedRows.containsKey(key.getRow())) {
+ log.error("Got unexpected key " + key);
+ } else {
+ expectedRows.put(key.getRow(), true);
+ }
+
+ count++;
+ }
+}
+
+/**
+ * Simple example for reading random batches of data from Accumulo. See docs/examples/README.batch for instructions.
+ */
+public class RandomBatchScanner {
+ private static final Logger log = Logger.getLogger(CountingVerifyingReceiver.class);
+
+ /**
+ * Generate a number of ranges, each covering a single random row.
+ *
+ * @param num
+ * the number of ranges to generate
+ * @param min
+ * the minimum row that will be generated
+ * @param max
+ * the maximum row that will be generated
+ * @param r
+ * a random number generator
+ * @param ranges
+ * a set in which to store the generated ranges
+ * @param expectedRows
+ * a map in which to store the rows covered by the ranges (initially mapped to false)
+ */
+ static void generateRandomQueries(int num, long min, long max, Random r, HashSet<Range> ranges, HashMap<Text,Boolean> expectedRows) {
+ log.info(String.format("Generating %,d random queries...", num));
+ while (ranges.size() < num) {
+ long rowid = (Math.abs(r.nextLong()) % (max - min)) + min;
+
+ Text row1 = new Text(String.format("row_%010d", rowid));
+
+ Range range = new Range(new Text(row1));
+ ranges.add(range);
+ expectedRows.put(row1, false);
+ }
+
+ log.info("finished");
+ }
+
+ /**
+ * Prints a count of the number of rows mapped to false.
+ *
+ * @param expectedRows
+ * @return boolean indicating "were all the rows found?"
+ */
+ private static boolean checkAllRowsFound(HashMap<Text,Boolean> expectedRows) {
+ int count = 0;
+ boolean allFound = true;
+ for (Entry<Text,Boolean> entry : expectedRows.entrySet())
+ if (!entry.getValue())
+ count++;
+
+ if (count > 0) {
+ log.warn("Did not find " + count + " rows");
+ allFound = false;
+ }
+ return allFound;
+ }
+
+ /**
+ * Generates a number of random queries, verifies that the key/value pairs returned were in the queried ranges and that the values were generated by
+ * {@link RandomBatchWriter#createValue(long, int)}. Prints information about the results.
+ *
+ * @param num
+ * the number of queries to generate
+ * @param min
+ * the min row to query
+ * @param max
+ * the max row to query
+ * @param evs
+ * the expected size of the values
+ * @param r
+ * a random number generator
+ * @param tsbr
+ * a batch scanner
+ * @return boolean indicating "did the queries go fine?"
+ */
+ static boolean doRandomQueries(int num, long min, long max, int evs, Random r, BatchScanner tsbr) {
+
+ HashSet<Range> ranges = new HashSet<Range>(num);
+ HashMap<Text,Boolean> expectedRows = new java.util.HashMap<Text,Boolean>();
+
+ generateRandomQueries(num, min, max, r, ranges, expectedRows);
+
+ tsbr.setRanges(ranges);
+
+ CountingVerifyingReceiver receiver = new CountingVerifyingReceiver(expectedRows, evs);
+
+ long t1 = System.currentTimeMillis();
+
+ for (Entry<Key,Value> entry : tsbr) {
+ receiver.receive(entry.getKey(), entry.getValue());
+ }
+
+ long t2 = System.currentTimeMillis();
+
+ log.info(String.format("%6.2f lookups/sec %6.2f secs%n", num / ((t2 - t1) / 1000.0), ((t2 - t1) / 1000.0)));
+ log.info(String.format("num results : %,d%n", receiver.count));
+
+ return checkAllRowsFound(expectedRows);
+ }
+
+ public static class Opts extends ClientOnRequiredTable {
+ @Parameter(names="--min", description="miniumum row that will be generated")
+ long min = 0;
+ @Parameter(names="--max", description="maximum ow that will be generated")
+ long max = 0;
+ @Parameter(names="--num", required=true, description="number of ranges to generate")
+ int num = 0;
+ @Parameter(names="--size", required=true, description="size of the value to write")
+ int size = 0;
+ @Parameter(names="--seed", description="seed for pseudo-random number generator")
+ Long seed = null;
+ }
+
+ /**
+ * Scans over a specified number of entries to Accumulo using a {@link BatchScanner}. Completes scans twice to compare times for a fresh query with those for
+ * a repeated query which has cached metadata and connections already established.
- *
- * @param args
- * @throws AccumuloException
- * @throws AccumuloSecurityException
- * @throws TableNotFoundException
+ */
+ public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ Opts opts = new Opts();
+ BatchScannerOpts bsOpts = new BatchScannerOpts();
+ opts.parseArgs(RandomBatchScanner.class.getName(), args, bsOpts);
+
+ Connector connector = opts.getConnector();
+ BatchScanner batchReader = connector.createBatchScanner(opts.tableName, opts.auths, bsOpts.scanThreads);
+ batchReader.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);
+
+ Random r;
+ if (opts.seed == null)
+ r = new Random();
+ else
+ r = new Random(opts.seed);
+
+ // do one cold
+ boolean status = doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader);
+
+ System.gc();
+ System.gc();
+ System.gc();
+
+ if (opts.seed == null)
+ r = new Random();
+ else
+ r = new Random(opts.seed);
+
+ // do one hot (connections already established, metadata table cached)
+ status = status && doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader);
+
+ batchReader.close();
+ if (!status) {
+ System.exit(1);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
index ce91da6,0000000..e76352a
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
@@@ -1,172 -1,0 +1,168 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.client;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.security.SecurityErrorCode;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Simple example for writing random data to Accumulo. See docs/examples/README.batch for instructions.
+ *
+ * The rows of the entries will be randomly generated numbers between a specified min and max (prefixed by "row_"). The column families will be "foo" and column
+ * qualifiers will be "1". The values will be random byte arrays of a specified size.
+ */
+public class RandomBatchWriter {
+
+ /**
+ * Creates a random byte array of specified size using the specified seed.
+ *
+ * @param rowid
+ * the seed to use for the random number generator
+ * @param dataSize
+ * the size of the array
+ * @return a random byte array
+ */
+ public static byte[] createValue(long rowid, int dataSize) {
+ Random r = new Random(rowid);
+ byte value[] = new byte[dataSize];
+
+ r.nextBytes(value);
+
+ // transform to printable chars
+ for (int j = 0; j < value.length; j++) {
+ value[j] = (byte) (((0xff & value[j]) % 92) + ' ');
+ }
+
+ return value;
+ }
+
+ /**
+ * Creates a mutation on a specified row with column family "foo", column qualifier "1", specified visibility, and a random value of specified size.
+ *
+ * @param rowid
+ * the row of the mutation
+ * @param dataSize
+ * the size of the random value
+ * @param visibility
+ * the visibility of the entry to insert
+ * @return a mutation
+ */
+ public static Mutation createMutation(long rowid, int dataSize, ColumnVisibility visibility) {
+ Text row = new Text(String.format("row_%010d", rowid));
+
+ Mutation m = new Mutation(row);
+
+ // create a random value that is a function of the
+ // row id for verification purposes
+ byte value[] = createValue(rowid, dataSize);
+
+ m.put(new Text("foo"), new Text("1"), visibility, new Value(value));
+
+ return m;
+ }
+
+ static class Opts extends ClientOnRequiredTable {
+ @Parameter(names="--num", required=true)
+ int num = 0;
+ @Parameter(names="--min")
+ long min = 0;
+ @Parameter(names="--max")
+ long max = Long.MAX_VALUE;
+ @Parameter(names="--size", required=true, description="size of the value to write")
+ int size = 0;
+ @Parameter(names="--vis", converter=VisibilityConverter.class)
+ ColumnVisibility visiblity = new ColumnVisibility("");
+ @Parameter(names="--seed", description="seed for pseudo-random number generator")
+ Long seed = null;
+ }
+
+ /**
+ * Writes a specified number of entries to Accumulo using a {@link BatchWriter}.
- *
- * @throws AccumuloException
- * @throws AccumuloSecurityException
- * @throws TableNotFoundException
+ */
+ public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ Opts opts = new Opts();
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(RandomBatchWriter.class.getName(), args, bwOpts);
+ if ((opts.max - opts.min) < opts.num) {
+ System.err.println(String.format("You must specify a min and a max that allow for at least num possible values. For example, you requested %d rows, but a min of %d and a max of %d only allows for %d rows.", opts.num, opts.min, opts.max, (opts.max - opts.min)));
+ System.exit(1);
+ }
+ Random r;
+ if (opts.seed == null)
+ r = new Random();
+ else {
+ r = new Random(opts.seed);
+ }
+ Connector connector = opts.getConnector();
+ BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
+
+ // reuse the ColumnVisibility object to improve performance
+ ColumnVisibility cv = opts.visiblity;
+
+ // Generate num unique row ids in the given range
+ HashSet<Long> rowids = new HashSet<Long>(opts.num);
+ while (rowids.size() < opts.num) {
+ rowids.add((Math.abs(r.nextLong()) % (opts.max - opts.min)) + opts.min);
+ }
+ for (long rowid : rowids) {
+ Mutation m = createMutation(rowid, opts.size, cv);
+ bw.addMutation(m);
+ }
+
+ try {
+ bw.close();
+ } catch (MutationsRejectedException e) {
+ if (e.getAuthorizationFailuresMap().size() > 0) {
+ HashMap<String,Set<SecurityErrorCode>> tables = new HashMap<String,Set<SecurityErrorCode>>();
+ for (Entry<KeyExtent,Set<SecurityErrorCode>> ke : e.getAuthorizationFailuresMap().entrySet()) {
+ Set<SecurityErrorCode> secCodes = tables.get(ke.getKey().getTableId().toString());
+ if (secCodes == null) {
+ secCodes = new HashSet<SecurityErrorCode>();
+ tables.put(ke.getKey().getTableId().toString(), secCodes);
+ }
+ secCodes.addAll(ke.getValue());
+ }
+ System.err.println("ERROR : Not authorized to write to tables : " + tables);
+ }
+
+ if (e.getConstraintViolationSummaries().size() > 0) {
+ System.err.println("ERROR : Constraint violations occurred : " + e.getConstraintViolationSummaries());
+ }
+ System.exit(1);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java
index a56fdc0,0000000..c37c1c3
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java
@@@ -1,73 -1,0 +1,68 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.client;
+
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.security.ColumnVisibility;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Simple example for writing random data in sequential order to Accumulo. See docs/examples/README.batch for instructions.
+ */
+public class SequentialBatchWriter {
+
+ static class Opts extends ClientOnRequiredTable {
+ @Parameter(names="--start")
+ long start = 0;
+ @Parameter(names="--num", required=true)
+ long num = 0;
+ @Parameter(names="--size", required=true, description="size of the value to write")
+ int valueSize = 0;
+ @Parameter(names="--vis", converter=VisibilityConverter.class)
+ ColumnVisibility vis = new ColumnVisibility();
+ }
+
+ /**
+ * Writes a specified number of entries to Accumulo using a {@link BatchWriter}. The rows of the entries will be sequential starting at a specified number.
+ * The column families will be "foo" and column qualifiers will be "1". The values will be random byte arrays of a specified size.
- *
- * @throws AccumuloException
- * @throws AccumuloSecurityException
- * @throws TableNotFoundException
- * @throws MutationsRejectedException
+ */
+ public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, MutationsRejectedException {
+ Opts opts = new Opts();
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(SequentialBatchWriter.class.getName(), args, bwOpts);
+ Connector connector = opts.getConnector();
+ BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
+
+ long end = opts.start + opts.num;
+
+ for (long i = opts.start; i < end; i++) {
+ Mutation m = RandomBatchWriter.createMutation(i, opts.valueSize, opts.vis);
+ bw.addMutation(m);
+ }
+
+ bw.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/TraceDumpExample.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/TraceDumpExample.java
index 2947e0e,0000000..70a23e5
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/TraceDumpExample.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/TraceDumpExample.java
@@@ -1,77 -1,0 +1,72 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.client;
+
+import org.apache.accumulo.core.cli.ClientOnDefaultTable;
+import org.apache.accumulo.core.cli.ScannerOpts;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.trace.TraceDump;
+import org.apache.accumulo.core.trace.TraceDump.Printer;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Example of using the TraceDump class to print a formatted view of a Trace
+ *
+ */
+public class TraceDumpExample {
+
+ static class Opts extends ClientOnDefaultTable {
+ public Opts() {
+ super("trace");
+ }
+
+ @Parameter(names = {"--traceid"}, description = "The hex string id of a given trace, for example 16cfbbd7beec4ae3")
+ public String traceId = "";
+ }
+
+ public void dump(Opts opts) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+
+ if (opts.traceId.isEmpty()) {
+ throw new IllegalArgumentException("--traceid option is required");
+ }
+
+ Scanner scanner = opts.getConnector().createScanner(opts.getTableName(), opts.auths);
+ scanner.setRange(new Range(new Text(opts.traceId)));
+ TraceDump.printTrace(scanner, new Printer() {
- public void print(String line) {
++ @Override
++ public void print(String line) {
+ System.out.println(line);
+ }
+ });
+ }
+
- /**
- * @param args
- * @throws AccumuloSecurityException
- * @throws AccumuloException
- * @throws TableNotFoundException
- */
+ public static void main(String[] args) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ TraceDumpExample traceDumpExample = new TraceDumpExample();
+ Opts opts = new Opts();
+ ScannerOpts scannerOpts = new ScannerOpts();
+ opts.parseArgs(TraceDumpExample.class.getName(), args, scannerOpts);
+
+ traceDumpExample.dump(opts);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java
index 744efed,0000000..f11c739
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java
@@@ -1,283 -1,0 +1,280 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.dirlist;
+
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Provides utility methods for getting the info for a file, listing the contents of a directory, and performing single wild card searches on file or directory
+ * names. See docs/examples/README.dirlist for instructions.
+ */
+public class QueryUtil {
+ private Connector conn = null;
+ private String tableName;
+ private Authorizations auths;
+ public static final Text DIR_COLF = new Text("dir");
+ public static final Text FORWARD_PREFIX = new Text("f");
+ public static final Text REVERSE_PREFIX = new Text("r");
+ public static final Text INDEX_COLF = new Text("i");
+ public static final Text COUNTS_COLQ = new Text("counts");
+
+ public QueryUtil(Opts opts) throws AccumuloException,
+ AccumuloSecurityException {
+ conn = opts.getConnector();
+ this.tableName = opts.tableName;
+ this.auths = opts.auths;
+ }
+
+ /**
+ * Calculates the depth of a path, i.e. the number of forward slashes in the path name.
+ *
+ * @param path
+ * the full path of a file or directory
+ * @return the depth of the path
+ */
+ public static int getDepth(String path) {
+ int numSlashes = 0;
+ int index = -1;
+ while ((index = path.indexOf("/", index + 1)) >= 0)
+ numSlashes++;
+ return numSlashes;
+ }
+
+ /**
+ * Given a path, construct an accumulo row prepended with the path's depth for the directory table.
+ *
+ * @param path
+ * the full path of a file or directory
+ * @return the accumulo row associated with this path
+ */
+ public static Text getRow(String path) {
+ Text row = new Text(String.format("%03d", getDepth(path)));
+ row.append(path.getBytes(), 0, path.length());
+ return row;
+ }
+
+ /**
+ * Given a path, construct an accumulo row prepended with the {@link #FORWARD_PREFIX} for the index table.
+ *
+ * @param path
+ * the full path of a file or directory
+ * @return the accumulo row associated with this path
+ */
+ public static Text getForwardIndex(String path) {
+ String part = path.substring(path.lastIndexOf("/") + 1);
+ if (part.length() == 0)
+ return null;
+ Text row = new Text(FORWARD_PREFIX);
+ row.append(part.getBytes(), 0, part.length());
+ return row;
+ }
+
+ /**
+ * Given a path, construct an accumulo row prepended with the {@link #REVERSE_PREFIX} with the path reversed for the index table.
+ *
+ * @param path
+ * the full path of a file or directory
+ * @return the accumulo row associated with this path
+ */
+ public static Text getReverseIndex(String path) {
+ String part = path.substring(path.lastIndexOf("/") + 1);
+ if (part.length() == 0)
+ return null;
+ byte[] rev = new byte[part.length()];
+ int i = part.length() - 1;
+ for (byte b : part.getBytes())
+ rev[i--] = b;
+ Text row = new Text(REVERSE_PREFIX);
+ row.append(rev, 0, rev.length);
+ return row;
+ }
+
+ /**
+ * Returns either the {@link #DIR_COLF} or a decoded string version of the colf.
+ *
+ * @param colf
+ * the column family
+ */
+ public static String getType(Text colf) {
+ if (colf.equals(DIR_COLF))
+ return colf.toString() + ":";
+ return Long.toString(Ingest.encoder.decode(colf.getBytes())) + ":";
+ }
+
+ /**
+ * Scans over the directory table and pulls out stat information about a path.
+ *
+ * @param path
+ * the full path of a file or directory
+ */
+ public Map<String,String> getData(String path) throws TableNotFoundException {
+ if (path.endsWith("/"))
+ path = path.substring(0, path.length() - 1);
+ Scanner scanner = conn.createScanner(tableName, auths);
+ scanner.setRange(new Range(getRow(path)));
+ Map<String,String> data = new TreeMap<String,String>();
+ for (Entry<Key,Value> e : scanner) {
+ String type = getType(e.getKey().getColumnFamily());
+ data.put("fullname", e.getKey().getRow().toString().substring(3));
+ data.put(type + e.getKey().getColumnQualifier().toString() + ":" + e.getKey().getColumnVisibility().toString(), new String(e.getValue().get()));
+ }
+ return data;
+ }
+
+ /**
+ * Uses the directory table to list the contents of a directory.
+ *
+ * @param path
+ * the full path of a directory
+ */
+ public Map<String,Map<String,String>> getDirList(String path) throws TableNotFoundException {
+ if (!path.endsWith("/"))
+ path = path + "/";
+ Map<String,Map<String,String>> fim = new TreeMap<String,Map<String,String>>();
+ Scanner scanner = conn.createScanner(tableName, auths);
+ scanner.setRange(Range.prefix(getRow(path)));
+ for (Entry<Key,Value> e : scanner) {
+ String name = e.getKey().getRow().toString();
+ name = name.substring(name.lastIndexOf("/") + 1);
+ String type = getType(e.getKey().getColumnFamily());
+ if (!fim.containsKey(name)) {
+ fim.put(name, new TreeMap<String,String>());
+ fim.get(name).put("fullname", e.getKey().getRow().toString().substring(3));
+ }
+ fim.get(name).put(type + e.getKey().getColumnQualifier().toString() + ":" + e.getKey().getColumnVisibility().toString(), new String(e.getValue().get()));
+ }
+ return fim;
+ }
+
+ /**
+ * Scans over the index table for files or directories with a given name.
+ *
+ * @param term
+ * the name a file or directory to search for
+ */
+ public Iterable<Entry<Key,Value>> exactTermSearch(String term) throws Exception {
+ System.out.println("executing exactTermSearch for " + term);
+ Scanner scanner = conn.createScanner(tableName, auths);
+ scanner.setRange(new Range(getForwardIndex(term)));
+ return scanner;
+ }
+
+ /**
+ * Scans over the index table for files or directories with a given name, prefix, or suffix (indicated by a wildcard '*' at the beginning or end of the term.
+ *
+ * @param exp
+ * the name a file or directory to search for with an optional wildcard '*' at the beginning or end
+ */
+ public Iterable<Entry<Key,Value>> singleRestrictedWildCardSearch(String exp) throws Exception {
+ if (exp.indexOf("/") >= 0)
+ throw new Exception("this method only works with unqualified names");
+
+ Scanner scanner = conn.createScanner(tableName, auths);
+ if (exp.startsWith("*")) {
+ System.out.println("executing beginning wildcard search for " + exp);
+ exp = exp.substring(1);
+ scanner.setRange(Range.prefix(getReverseIndex(exp)));
+ } else if (exp.endsWith("*")) {
+ System.out.println("executing ending wildcard search for " + exp);
+ exp = exp.substring(0, exp.length() - 1);
+ scanner.setRange(Range.prefix(getForwardIndex(exp)));
+ } else if (exp.indexOf("*") >= 0) {
+ throw new Exception("this method only works for beginning or ending wild cards");
+ } else {
+ return exactTermSearch(exp);
+ }
+ return scanner;
+ }
+
+ /**
+ * Scans over the index table for files or directories with a given name that can contain a single wildcard '*' anywhere in the term.
+ *
+ * @param exp
+ * the name a file or directory to search for with one optional wildcard '*'
+ */
+ public Iterable<Entry<Key,Value>> singleWildCardSearch(String exp) throws Exception {
+ int starIndex = exp.indexOf("*");
+ if (exp.indexOf("*", starIndex + 1) >= 0)
+ throw new Exception("only one wild card for search");
+
+ if (starIndex < 0) {
+ return exactTermSearch(exp);
+ } else if (starIndex == 0 || starIndex == exp.length() - 1) {
+ return singleRestrictedWildCardSearch(exp);
+ }
+
+ String firstPart = exp.substring(0, starIndex);
+ String lastPart = exp.substring(starIndex + 1);
+ String regexString = ".*/" + exp.replace("*", "[^/]*");
+
+ Scanner scanner = conn.createScanner(tableName, auths);
+ if (firstPart.length() >= lastPart.length()) {
+ System.out.println("executing middle wildcard search for " + regexString + " from entries starting with " + firstPart);
+ scanner.setRange(Range.prefix(getForwardIndex(firstPart)));
+ } else {
+ System.out.println("executing middle wildcard search for " + regexString + " from entries ending with " + lastPart);
+ scanner.setRange(Range.prefix(getReverseIndex(lastPart)));
+ }
+ IteratorSetting regex = new IteratorSetting(50, "regex", RegExFilter.class);
+ RegExFilter.setRegexs(regex, null, null, regexString, null, false);
+ scanner.addScanIterator(regex);
+ return scanner;
+ }
+
+ public static class Opts extends ClientOnRequiredTable {
+ @Parameter(names="--path", description="the directory to list")
+ String path = "/";
+ @Parameter(names="--search", description="find a file or directorys with the given name")
+ boolean search = false;
+ }
+
+ /**
+ * Lists the contents of a directory using the directory table, or searches for file or directory names (if the -search flag is included).
- *
- * @param args
- * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(QueryUtil.class.getName(), args);
+ QueryUtil q = new QueryUtil(opts);
+ if (opts.search) {
+ for (Entry<Key,Value> e : q.singleWildCardSearch(opts.path)) {
+ System.out.println(e.getKey().getColumnQualifier());
+ }
+ } else {
+ for (Entry<String,Map<String,String>> e : q.getDirList(opts.path).entrySet()) {
+ System.out.println(e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/NGramIngest.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/NGramIngest.java
index cd6ca40,0000000..dc14512
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/NGramIngest.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/NGramIngest.java
@@@ -1,116 -1,0 +1,113 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.mapreduce;
+
+import java.io.IOException;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Map job to ingest n-gram files from
+ * http://storage.googleapis.com/books/ngrams/books/datasetsv2.html
+ */
+public class NGramIngest extends Configured implements Tool {
+
+ private static final Logger log = Logger.getLogger(NGramIngest.class);
+
+
+ static class Opts extends ClientOnRequiredTable {
+ @Parameter(names = "--input", required=true)
+ String inputDirectory;
+ }
+ static class NGramMapper extends Mapper<LongWritable, Text, Text, Mutation> {
+
+ @Override
+ protected void map(LongWritable location, Text value, Context context) throws IOException, InterruptedException {
+ String parts[] = value.toString().split("\\t");
+ if (parts.length >= 4) {
+ Mutation m = new Mutation(parts[0]);
+ m.put(parts[1], String.format("%010d", Long.parseLong(parts[2])), new Value(parts[3].trim().getBytes()));
+ context.write(null, m);
+ }
+ }
+ }
+
- /**
- * @param args
- */
+ @Override
+ public int run(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(getClass().getName(), args);
+
+ Job job = new Job(getConf(), getClass().getSimpleName());
+ job.setJarByClass(getClass());
+
+ opts.setAccumuloConfigs(job);
+ job.setInputFormatClass(TextInputFormat.class);
+ job.setOutputFormatClass(AccumuloOutputFormat.class);
+
+ job.setMapperClass(NGramMapper.class);
+ job.setMapOutputKeyClass(Text.class);
+ job.setMapOutputValueClass(Mutation.class);
+
+ job.setNumReduceTasks(0);
+ job.setSpeculativeExecution(false);
+
+
+ if (!opts.getConnector().tableOperations().exists(opts.tableName)) {
+ log.info("Creating table " + opts.tableName);
+ opts.getConnector().tableOperations().create(opts.tableName);
+ SortedSet<Text> splits = new TreeSet<Text>();
+ String numbers[] = "1 2 3 4 5 6 7 8 9".split("\\s");
+ String lower[] = "a b c d e f g h i j k l m n o p q r s t u v w x y z".split("\\s");
+ String upper[] = "A B C D E F G H I J K L M N O P Q R S T U V W X Y Z".split("\\s");
+ for (String[] array : new String[][]{numbers, lower, upper}) {
+ for (String s : array) {
+ splits.add(new Text(s));
+ }
+ }
+ opts.getConnector().tableOperations().addSplits(opts.tableName, splits);
+ }
+
+ TextInputFormat.addInputPath(job, new Path(opts.inputDirectory));
+ job.waitForCompletion(true);
+ return job.isSuccessful() ? 0 : 1;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(CachedConfiguration.getInstance(), new NGramIngest(), args);
+ if (res != 0)
+ System.exit(res);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
index d8eedef,0000000..669c76d
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
@@@ -1,130 -1,0 +1,129 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.mapreduce;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Map;
+
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.format.DefaultFormatter;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Takes a table and outputs the specified column to a set of part files on hdfs accumulo accumulo.examples.mapreduce.TableToFile <username> <password>
+ * <tablename> <column> <hdfs-output-path>
+ */
+public class TableToFile extends Configured implements Tool {
+
+ static class Opts extends ClientOnRequiredTable {
+ @Parameter(names = "--output", description = "output directory", required = true)
+ String output;
+ @Parameter(names = "--columns", description = "columns to extract, in cf:cq{,cf:cq,...} form")
+ String columns = "";
+ }
+
+ /**
+ * The Mapper class that given a row number, will generate the appropriate output line.
+ */
+ public static class TTFMapper extends Mapper<Key,Value,NullWritable,Text> {
+ @Override
+ public void map(Key row, Value data, Context context) throws IOException, InterruptedException {
+ final Key r = row;
+ final Value v = data;
+ Map.Entry<Key,Value> entry = new Map.Entry<Key,Value>() {
+ @Override
+ public Key getKey() {
+ return r;
+ }
+
+ @Override
+ public Value getValue() {
+ return v;
+ }
+
+ @Override
+ public Value setValue(Value value) {
+ return null;
+ }
+ };
+ context.write(NullWritable.get(), new Text(DefaultFormatter.formatEntry(entry, false)));
+ context.setStatus("Outputed Value");
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException, AccumuloSecurityException {
+ Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+ job.setJarByClass(this.getClass());
+ Opts opts = new Opts();
+ opts.parseArgs(getClass().getName(), args);
+
+ job.setInputFormatClass(AccumuloInputFormat.class);
+ opts.setAccumuloConfigs(job);
+
+ HashSet<Pair<Text,Text>> columnsToFetch = new HashSet<Pair<Text,Text>>();
+ for (String col : opts.columns.split(",")) {
+ int idx = col.indexOf(":");
+ Text cf = new Text(idx < 0 ? col : col.substring(0, idx));
+ Text cq = idx < 0 ? null : new Text(col.substring(idx + 1));
+ if (cf.getLength() > 0)
+ columnsToFetch.add(new Pair<Text,Text>(cf, cq));
+ }
+ if (!columnsToFetch.isEmpty())
+ AccumuloInputFormat.fetchColumns(job, columnsToFetch);
+
+ job.setMapperClass(TTFMapper.class);
+ job.setMapOutputKeyClass(NullWritable.class);
+ job.setMapOutputValueClass(Text.class);
+
+ job.setNumReduceTasks(0);
+
+ job.setOutputFormatClass(TextOutputFormat.class);
+ TextOutputFormat.setOutputPath(job, new Path(opts.output));
+
+ job.waitForCompletion(true);
+ return job.isSuccessful() ? 0 : 1;
+ }
+
+ /**
+ *
+ * @param args
+ * instanceName zookeepers username password table columns outputpath
- * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(CachedConfiguration.getInstance(), new TableToFile(), args);
+ if (res != 0)
+ System.exit(res);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
index f6d610e,0000000..d98d78b
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
@@@ -1,78 -1,0 +1,75 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.shard;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.cli.BatchScannerOpts;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.IntersectingIterator;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * This program queries a set of terms in the shard table (populated by {@link Index}) using the {@link IntersectingIterator}.
+ *
+ * See docs/examples/README.shard for instructions.
+ */
+
+public class Query {
+
+ static class Opts extends ClientOnRequiredTable {
+ @Parameter(description=" term { <term> ... }")
+ List<String> terms = new ArrayList<String>();
+ }
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ BatchScannerOpts bsOpts = new BatchScannerOpts();
+ opts.parseArgs(Query.class.getName(), args, bsOpts);
+
+ Connector conn = opts.getConnector();
+ BatchScanner bs = conn.createBatchScanner(opts.tableName, opts.auths, bsOpts.scanThreads);
+ bs.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);
+
+ Text columns[] = new Text[opts.terms.size()];
+ int i = 0;
+ for (String term : opts.terms) {
+ columns[i++] = new Text(term);
+ }
+ IteratorSetting ii = new IteratorSetting(20, "ii", IntersectingIterator.class);
+ IntersectingIterator.setColumnFamilies(ii, columns);
+ bs.addScanIterator(ii);
+ bs.setRanges(Collections.singleton(new Range()));
+ for (Entry<Key,Value> entry : bs) {
+ System.out.println(" " + entry.getKey().getColumnQualifier());
+ }
+
+ }
+
+}
[44/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFile.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFile.java
index f2cb326,0000000..b7a927b
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFile.java
@@@ -1,2030 -1,0 +1,1986 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.accumulo.core.file.rfile.bcfile;
+
+import java.io.ByteArrayInputStream;
+import java.io.Closeable;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Comparator;
+
+import org.apache.accumulo.core.file.rfile.bcfile.BCFile.Reader.BlockReader;
+import org.apache.accumulo.core.file.rfile.bcfile.BCFile.Writer.BlockAppender;
+import org.apache.accumulo.core.file.rfile.bcfile.Chunk.ChunkDecoder;
+import org.apache.accumulo.core.file.rfile.bcfile.Chunk.ChunkEncoder;
+import org.apache.accumulo.core.file.rfile.bcfile.CompareUtils.BytesComparator;
+import org.apache.accumulo.core.file.rfile.bcfile.CompareUtils.MemcmpRawComparator;
+import org.apache.accumulo.core.file.rfile.bcfile.Utils.Version;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.RawComparator;
+import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.io.serializer.JavaSerializationComparator;
+
+/**
+ * A TFile is a container of key-value pairs. Both keys and values are type-less bytes. Keys are restricted to 64KB, value length is not restricted (practically
+ * limited to the available disk storage). TFile further provides the following features:
+ * <ul>
+ * <li>Block Compression.
+ * <li>Named meta data blocks.
+ * <li>Sorted or unsorted keys.
+ * <li>Seek by key or by file offset.
+ * </ul>
+ * The memory footprint of a TFile includes the following:
+ * <ul>
+ * <li>Some constant overhead of reading or writing a compressed block.
+ * <ul>
+ * <li>Each compressed block requires one compression/decompression codec for I/O.
+ * <li>Temporary space to buffer the key.
+ * <li>Temporary space to buffer the value (for TFile.Writer only). Values are chunk encoded, so that we buffer at most one chunk of user data. By default, the
+ * chunk buffer is 1MB. Reading chunked value does not require additional memory.
+ * </ul>
+ * <li>TFile index, which is proportional to the total number of Data Blocks. The total amount of memory needed to hold the index can be estimated as
+ * (56+AvgKeySize)*NumBlocks.
+ * <li>MetaBlock index, which is proportional to the total number of Meta Blocks.The total amount of memory needed to hold the index for Meta Blocks can be
+ * estimated as (40+AvgMetaBlockName)*NumMetaBlock.
+ * </ul>
+ * <p>
+ * The behavior of TFile can be customized by the following variables through Configuration:
+ * <ul>
+ * <li><b>tfile.io.chunk.size</b>: Value chunk size. Integer (in bytes). Default to 1MB. Values of the length less than the chunk size is guaranteed to have
+ * known value length in read time (See {@link TFile.Reader.Scanner.Entry#isValueLengthKnown()}).
+ * <li><b>tfile.fs.output.buffer.size</b>: Buffer size used for FSDataOutputStream. Integer (in bytes). Default to 256KB.
+ * <li><b>tfile.fs.input.buffer.size</b>: Buffer size used for FSDataInputStream. Integer (in bytes). Default to 256KB.
+ * </ul>
+ * <p>
+ * Suggestions on performance optimization.
+ * <ul>
+ * <li>Minimum block size. We recommend a setting of minimum block size between 256KB to 1MB for general usage. Larger block size is preferred if files are
+ * primarily for sequential access. However, it would lead to inefficient random access (because there are more data to decompress). Smaller blocks are good for
+ * random access, but require more memory to hold the block index, and may be slower to create (because we must flush the compressor stream at the conclusion of
+ * each data block, which leads to an FS I/O flush). Further, due to the internal caching in Compression codec, the smallest possible block size would be around
+ * 20KB-30KB.
+ * <li>The current implementation does not offer true multi-threading for reading. The implementation uses FSDataInputStream seek()+read(), which is shown to be
+ * much faster than positioned-read call in single thread mode. However, it also means that if multiple threads attempt to access the same TFile (using multiple
+ * scanners) simultaneously, the actual I/O is carried out sequentially even if they access different DFS blocks.
+ * <li>Compression codec. Use "none" if the data is not very compressable (by compressable, I mean a compression ratio at least 2:1). Generally, use "lzo" as
+ * the starting point for experimenting. "gz" overs slightly better compression ratio over "lzo" but requires 4x CPU to compress and 2x CPU to decompress,
+ * comparing to "lzo".
+ * <li>File system buffering, if the underlying FSDataInputStream and FSDataOutputStream is already adequately buffered; or if applications reads/writes keys
+ * and values in large buffers, we can reduce the sizes of input/output buffering in TFile layer by setting the configuration parameters
+ * "tfile.fs.input.buffer.size" and "tfile.fs.output.buffer.size".
+ * </ul>
+ *
+ * Some design rationale behind TFile can be found at <a href=https://issues.apache.org/jira/browse/HADOOP-3315>Hadoop-3315</a>.
+ */
+public class TFile {
+ static final Log LOG = LogFactory.getLog(TFile.class);
+
+ private static final String CHUNK_BUF_SIZE_ATTR = "tfile.io.chunk.size";
+ private static final String FS_INPUT_BUF_SIZE_ATTR = "tfile.fs.input.buffer.size";
+ private static final String FS_OUTPUT_BUF_SIZE_ATTR = "tfile.fs.output.buffer.size";
+
+ static int getChunkBufferSize(Configuration conf) {
+ int ret = conf.getInt(CHUNK_BUF_SIZE_ATTR, 1024 * 1024);
+ return (ret > 0) ? ret : 1024 * 1024;
+ }
+
+ static int getFSInputBufferSize(Configuration conf) {
+ return conf.getInt(FS_INPUT_BUF_SIZE_ATTR, 256 * 1024);
+ }
+
+ static int getFSOutputBufferSize(Configuration conf) {
+ return conf.getInt(FS_OUTPUT_BUF_SIZE_ATTR, 256 * 1024);
+ }
+
+ private static final int MAX_KEY_SIZE = 64 * 1024; // 64KB
+ static final Version API_VERSION = new Version((short) 1, (short) 0);
+
+ /** snappy codec **/
+ public static final String COMPRESSION_SNAPPY = "snappy";
+
+ /** compression: gzip */
+ public static final String COMPRESSION_GZ = "gz";
+ /** compression: lzo */
+ public static final String COMPRESSION_LZO = "lzo";
+ /** compression: none */
+ public static final String COMPRESSION_NONE = "none";
+ /** comparator: memcmp */
+ public static final String COMPARATOR_MEMCMP = "memcmp";
+ /** comparator prefix: java class */
+ public static final String COMPARATOR_JCLASS = "jclass:";
+
+ /**
+ * Make a raw comparator from a string name.
+ *
+ * @param name
+ * Comparator name
+ * @return A RawComparable comparator.
+ */
+ static public Comparator<RawComparable> makeComparator(String name) {
+ return TFileMeta.makeComparator(name);
+ }
+
+ // Prevent the instantiation of TFiles
+ private TFile() {
+ // nothing
+ }
+
+ /**
+ * Get names of supported compression algorithms. The names are acceptable by TFile.Writer.
+ *
+ * @return Array of strings, each represents a supported compression algorithm. Currently, the following compression algorithms are supported.
+ * <ul>
+ * <li>"none" - No compression.
+ * <li>"lzo" - LZO compression.
+ * <li>"gz" - GZIP compression.
+ * <li>"snappy" - Snappy compression
+ * </ul>
+ */
+ public static String[] getSupportedCompressionAlgorithms() {
+ return Compression.getSupportedAlgorithms();
+ }
+
+ /**
+ * TFile Writer.
+ */
+ public static class Writer implements Closeable {
+ // minimum compressed size for a block.
+ private final int sizeMinBlock;
+
+ // Meta blocks.
+ final TFileIndex tfileIndex;
+ final TFileMeta tfileMeta;
+
+ // reference to the underlying BCFile.
+ private BCFile.Writer writerBCF;
+
+ // current data block appender.
+ BlockAppender blkAppender;
+ long blkRecordCount;
+
+ // buffers for caching the key.
+ BoundedByteArrayOutputStream currentKeyBufferOS;
+ BoundedByteArrayOutputStream lastKeyBufferOS;
+
+ // buffer used by chunk codec
+ private byte[] valueBuffer;
+
+ /**
+ * Writer states. The state always transits in circles: READY -> IN_KEY -> END_KEY -> IN_VALUE -> READY.
+ */
+ private enum State {
+ READY, // Ready to start a new key-value pair insertion.
+ IN_KEY, // In the middle of key insertion.
+ END_KEY, // Key insertion complete, ready to insert value.
+ IN_VALUE, // In value insertion.
+ // ERROR, // Error encountered, cannot continue.
+ CLOSED, // TFile already closed.
+ }
+
+ // current state of Writer.
+ State state = State.READY;
+ Configuration conf;
+ long errorCount = 0;
+
+ /**
+ * Constructor
+ *
+ * @param fsdos
+ * output stream for writing. Must be at position 0.
+ * @param minBlockSize
+ * Minimum compressed block size in bytes. A compression block will not be closed until it reaches this size except for the last block.
+ * @param compressName
+ * Name of the compression algorithm. Must be one of the strings returned by {@link TFile#getSupportedCompressionAlgorithms()}.
+ * @param comparator
+ * Leave comparator as null or empty string if TFile is not sorted. Otherwise, provide the string name for the comparison algorithm for keys. Two
+ * kinds of comparators are supported.
+ * <ul>
+ * <li>Algorithmic comparator: binary comparators that is language independent. Currently, only "memcmp" is supported.
+ * <li>Language-specific comparator: binary comparators that can only be constructed in specific language. For Java, the syntax is "jclass:",
+ * followed by the class name of the RawComparator. Currently, we only support RawComparators that can be constructed through the default
+ * constructor (with no parameters). Parameterized RawComparators such as {@link WritableComparator} or {@link JavaSerializationComparator} may not
+ * be directly used. One should write a wrapper class that inherits from such classes and use its default constructor to perform proper
+ * initialization.
+ * </ul>
+ * @param conf
+ * The configuration object.
- * @throws IOException
+ */
+ public Writer(FSDataOutputStream fsdos, int minBlockSize, String compressName, String comparator, Configuration conf) throws IOException {
+ sizeMinBlock = minBlockSize;
+ tfileMeta = new TFileMeta(comparator);
+ tfileIndex = new TFileIndex(tfileMeta.getComparator());
+
+ writerBCF = new BCFile.Writer(fsdos, compressName, conf, true);
+ currentKeyBufferOS = new BoundedByteArrayOutputStream(MAX_KEY_SIZE);
+ lastKeyBufferOS = new BoundedByteArrayOutputStream(MAX_KEY_SIZE);
+ this.conf = conf;
+ }
+
+ /**
+ * Close the Writer. Resources will be released regardless of the exceptions being thrown. Future close calls will have no effect.
+ *
+ * The underlying FSDataOutputStream is not closed.
+ */
+ public void close() throws IOException {
+ if ((state == State.CLOSED)) {
+ return;
+ }
+ try {
+ // First try the normal finish.
+ // Terminate upon the first Exception.
+ if (errorCount == 0) {
+ if (state != State.READY) {
+ throw new IllegalStateException("Cannot close TFile in the middle of key-value insertion.");
+ }
+
+ finishDataBlock(true);
+
+ // first, write out data:TFile.meta
+ BlockAppender outMeta = writerBCF.prepareMetaBlock(TFileMeta.BLOCK_NAME, COMPRESSION_NONE);
+ try {
+ tfileMeta.write(outMeta);
+ } finally {
+ outMeta.close();
+ }
+
+ // second, write out data:TFile.index
+ BlockAppender outIndex = writerBCF.prepareMetaBlock(TFileIndex.BLOCK_NAME);
+ try {
+ tfileIndex.write(outIndex);
+ } finally {
+ outIndex.close();
+ }
+
+ writerBCF.close();
+ }
+ } finally {
+ IOUtils.cleanup(LOG, blkAppender, writerBCF);
+ blkAppender = null;
+ writerBCF = null;
+ state = State.CLOSED;
+ }
+ }
+
+ /**
+ * Adding a new key-value pair to the TFile. This is synonymous to append(key, 0, key.length, value, 0, value.length)
+ *
+ * @param key
+ * Buffer for key.
+ * @param value
+ * Buffer for value.
- * @throws IOException
+ */
+ public void append(byte[] key, byte[] value) throws IOException {
+ append(key, 0, key.length, value, 0, value.length);
+ }
+
+ /**
+ * Adding a new key-value pair to TFile.
+ *
+ * @param key
+ * buffer for key.
+ * @param koff
+ * offset in key buffer.
+ * @param klen
+ * length of key.
+ * @param value
+ * buffer for value.
+ * @param voff
+ * offset in value buffer.
+ * @param vlen
+ * length of value.
+ * @throws IOException
+ * Upon IO errors.
+ * <p>
+ * If an exception is thrown, the TFile will be in an inconsistent state. The only legitimate call after that would be close
+ */
+ public void append(byte[] key, int koff, int klen, byte[] value, int voff, int vlen) throws IOException {
+ if ((koff | klen | (koff + klen) | (key.length - (koff + klen))) < 0) {
+ throw new IndexOutOfBoundsException("Bad key buffer offset-length combination.");
+ }
+
+ if ((voff | vlen | (voff + vlen) | (value.length - (voff + vlen))) < 0) {
+ throw new IndexOutOfBoundsException("Bad value buffer offset-length combination.");
+ }
+
+ try {
+ DataOutputStream dosKey = prepareAppendKey(klen);
+ try {
+ ++errorCount;
+ dosKey.write(key, koff, klen);
+ --errorCount;
+ } finally {
+ dosKey.close();
+ }
+
+ DataOutputStream dosValue = prepareAppendValue(vlen);
+ try {
+ ++errorCount;
+ dosValue.write(value, voff, vlen);
+ --errorCount;
+ } finally {
+ dosValue.close();
+ }
+ } finally {
+ state = State.READY;
+ }
+ }
+
+ /**
+ * Helper class to register key after close call on key append stream.
+ */
+ private class KeyRegister extends DataOutputStream {
+ private final int expectedLength;
+ private boolean closed = false;
+
+ public KeyRegister(int len) {
+ super(currentKeyBufferOS);
+ if (len >= 0) {
+ currentKeyBufferOS.reset(len);
+ } else {
+ currentKeyBufferOS.reset();
+ }
+ expectedLength = len;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed == true) {
+ return;
+ }
+
+ try {
+ ++errorCount;
+ byte[] key = currentKeyBufferOS.getBuffer();
+ int len = currentKeyBufferOS.size();
+ /**
+ * verify length.
+ */
+ if (expectedLength >= 0 && expectedLength != len) {
+ throw new IOException("Incorrect key length: expected=" + expectedLength + " actual=" + len);
+ }
+
+ Utils.writeVInt(blkAppender, len);
+ blkAppender.write(key, 0, len);
+ if (tfileIndex.getFirstKey() == null) {
+ tfileIndex.setFirstKey(key, 0, len);
+ }
+
+ if (tfileMeta.isSorted()) {
+ byte[] lastKey = lastKeyBufferOS.getBuffer();
+ int lastLen = lastKeyBufferOS.size();
+ if (tfileMeta.getComparator().compare(key, 0, len, lastKey, 0, lastLen) < 0) {
+ throw new IOException("Keys are not added in sorted order");
+ }
+ }
+
+ BoundedByteArrayOutputStream tmp = currentKeyBufferOS;
+ currentKeyBufferOS = lastKeyBufferOS;
+ lastKeyBufferOS = tmp;
+ --errorCount;
+ } finally {
+ closed = true;
+ state = State.END_KEY;
+ }
+ }
+ }
+
+ /**
+ * Helper class to register value after close call on value append stream.
+ */
+ private class ValueRegister extends DataOutputStream {
+ private boolean closed = false;
+
+ public ValueRegister(OutputStream os) {
+ super(os);
+ }
+
+ // Avoiding flushing call to down stream.
+ @Override
+ public void flush() {
+ // do nothing
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed == true) {
+ return;
+ }
+
+ try {
+ ++errorCount;
+ super.close();
+ blkRecordCount++;
+ // bump up the total record count in the whole file
+ tfileMeta.incRecordCount();
+ finishDataBlock(false);
+ --errorCount;
+ } finally {
+ closed = true;
+ state = State.READY;
+ }
+ }
+ }
+
+ /**
+ * Obtain an output stream for writing a key into TFile. This may only be called when there is no active Key appending stream or value appending stream.
+ *
+ * @param length
+ * The expected length of the key. If length of the key is not known, set length = -1. Otherwise, the application must write exactly as many bytes
+ * as specified here before calling close on the returned output stream.
+ * @return The key appending output stream.
- * @throws IOException
+ *
+ */
+ public DataOutputStream prepareAppendKey(int length) throws IOException {
+ if (state != State.READY) {
+ throw new IllegalStateException("Incorrect state to start a new key: " + state.name());
+ }
+
+ initDataBlock();
+ DataOutputStream ret = new KeyRegister(length);
+ state = State.IN_KEY;
+ return ret;
+ }
+
+ /**
+ * Obtain an output stream for writing a value into TFile. This may only be called right after a key appending operation (the key append stream must be
+ * closed).
+ *
+ * @param length
+ * The expected length of the value. If length of the value is not known, set length = -1. Otherwise, the application must write exactly as many
+ * bytes as specified here before calling close on the returned output stream. Advertising the value size up-front guarantees that the value is
+ * encoded in one chunk, and avoids intermediate chunk buffering.
- * @throws IOException
- *
+ */
+ public DataOutputStream prepareAppendValue(int length) throws IOException {
+ if (state != State.END_KEY) {
+ throw new IllegalStateException("Incorrect state to start a new value: " + state.name());
+ }
+
+ DataOutputStream ret;
+
+ // unknown length
+ if (length < 0) {
+ if (valueBuffer == null) {
+ valueBuffer = new byte[getChunkBufferSize(conf)];
+ }
+ ret = new ValueRegister(new ChunkEncoder(blkAppender, valueBuffer));
+ } else {
+ ret = new ValueRegister(new Chunk.SingleChunkEncoder(blkAppender, length));
+ }
+
+ state = State.IN_VALUE;
+ return ret;
+ }
+
+ /**
+ * Obtain an output stream for creating a meta block. This function may not be called when there is a key append stream or value append stream active. No
+ * more key-value insertion is allowed after a meta data block has been added to TFile.
+ *
+ * @param name
+ * Name of the meta block.
+ * @param compressName
+ * Name of the compression algorithm to be used. Must be one of the strings returned by {@link TFile#getSupportedCompressionAlgorithms()}.
+ * @return A DataOutputStream that can be used to write Meta Block data. Closing the stream would signal the ending of the block.
- * @throws IOException
+ * @throws MetaBlockAlreadyExists
+ * the Meta Block with the same name already exists.
+ */
+ public DataOutputStream prepareMetaBlock(String name, String compressName) throws IOException, MetaBlockAlreadyExists {
+ if (state != State.READY) {
+ throw new IllegalStateException("Incorrect state to start a Meta Block: " + state.name());
+ }
+
+ finishDataBlock(true);
+ DataOutputStream outputStream = writerBCF.prepareMetaBlock(name, compressName);
+ return outputStream;
+ }
+
+ /**
+ * Obtain an output stream for creating a meta block. This function may not be called when there is a key append stream or value append stream active. No
+ * more key-value insertion is allowed after a meta data block has been added to TFile. Data will be compressed using the default compressor as defined in
+ * Writer's constructor.
+ *
+ * @param name
+ * Name of the meta block.
+ * @return A DataOutputStream that can be used to write Meta Block data. Closing the stream would signal the ending of the block.
- * @throws IOException
+ * @throws MetaBlockAlreadyExists
+ * the Meta Block with the same name already exists.
+ */
+ public DataOutputStream prepareMetaBlock(String name) throws IOException, MetaBlockAlreadyExists {
+ if (state != State.READY) {
+ throw new IllegalStateException("Incorrect state to start a Meta Block: " + state.name());
+ }
+
+ finishDataBlock(true);
+ return writerBCF.prepareMetaBlock(name);
+ }
+
+ /**
+ * Check if we need to start a new data block.
+ *
+ * @throws IOException
+ */
+ private void initDataBlock() throws IOException {
+ // for each new block, get a new appender
+ if (blkAppender == null) {
+ blkAppender = writerBCF.prepareDataBlock();
+ }
+ }
+
+ /**
+ * Close the current data block if necessary.
+ *
+ * @param bForceFinish
+ * Force the closure regardless of the block size.
+ * @throws IOException
+ */
+ void finishDataBlock(boolean bForceFinish) throws IOException {
+ if (blkAppender == null) {
+ return;
+ }
+
+ // exceeded the size limit, do the compression and finish the block
+ if (bForceFinish || blkAppender.getCompressedSize() >= sizeMinBlock) {
+ // keep tracks of the last key of each data block, no padding
+ // for now
+ TFileIndexEntry keyLast = new TFileIndexEntry(lastKeyBufferOS.getBuffer(), 0, lastKeyBufferOS.size(), blkRecordCount);
+ tfileIndex.addEntry(keyLast);
+ // close the appender
+ blkAppender.close();
+ blkAppender = null;
+ blkRecordCount = 0;
+ }
+ }
+ }
+
+ /**
+ * TFile Reader. Users may only read TFiles by creating TFile.Reader.Scanner. objects. A scanner may scan the whole TFile ({@link Reader#createScanner()} ) ,
+ * a portion of TFile based on byte offsets ( {@link Reader#createScanner(long, long)}), or a portion of TFile with keys fall in a certain key range (for
+ * sorted TFile only, {@link Reader#createScanner(byte[], byte[])} or {@link Reader#createScanner(RawComparable, RawComparable)}).
+ */
+ public static class Reader implements Closeable {
+ // The underlying BCFile reader.
+ final BCFile.Reader readerBCF;
+
+ // TFile index, it is loaded lazily.
+ TFileIndex tfileIndex = null;
+ final TFileMeta tfileMeta;
+ final BytesComparator comparator;
+
+ // global begin and end locations.
+ private final Location begin;
+ private final Location end;
+
+ /**
+ * Location representing a virtual position in the TFile.
+ */
+ static final class Location implements Comparable<Location>, Cloneable {
+ private int blockIndex;
+ // distance/offset from the beginning of the block
+ private long recordIndex;
+
+ Location(int blockIndex, long recordIndex) {
+ set(blockIndex, recordIndex);
+ }
+
+ void incRecordIndex() {
+ ++recordIndex;
+ }
+
+ Location(Location other) {
+ set(other);
+ }
+
+ int getBlockIndex() {
+ return blockIndex;
+ }
+
+ long getRecordIndex() {
+ return recordIndex;
+ }
+
+ void set(int blockIndex, long recordIndex) {
+ if ((blockIndex | recordIndex) < 0) {
+ throw new IllegalArgumentException("Illegal parameter for BlockLocation.");
+ }
+ this.blockIndex = blockIndex;
+ this.recordIndex = recordIndex;
+ }
+
+ void set(Location other) {
+ set(other.blockIndex, other.recordIndex);
+ }
+
+ /**
+ * @see java.lang.Comparable#compareTo(java.lang.Object)
+ */
+ @Override
+ public int compareTo(Location other) {
+ return compareTo(other.blockIndex, other.recordIndex);
+ }
+
+ int compareTo(int bid, long rid) {
+ if (this.blockIndex == bid) {
+ long ret = this.recordIndex - rid;
+ if (ret > 0)
+ return 1;
+ if (ret < 0)
+ return -1;
+ return 0;
+ }
+ return this.blockIndex - bid;
+ }
+
+ /**
+ * @see java.lang.Object#clone()
+ */
+ @Override
+ protected Location clone() {
+ return new Location(blockIndex, recordIndex);
+ }
+
+ /**
+ * @see java.lang.Object#hashCode()
+ */
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = prime + blockIndex;
+ result = (int) (prime * result + recordIndex);
+ return result;
+ }
+
+ /**
+ * @see java.lang.Object#equals(java.lang.Object)
+ */
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ Location other = (Location) obj;
+ if (blockIndex != other.blockIndex)
+ return false;
+ if (recordIndex != other.recordIndex)
+ return false;
+ return true;
+ }
+ }
+
+ /**
+ * Constructor
+ *
+ * @param fsdis
+ * FS input stream of the TFile.
+ * @param fileLength
+ * The length of TFile. This is required because we have no easy way of knowing the actual size of the input file through the File input stream.
- * @param conf
- * @throws IOException
+ */
+ public Reader(FSDataInputStream fsdis, long fileLength, Configuration conf) throws IOException {
+ readerBCF = new BCFile.Reader(fsdis, fileLength, conf);
+
+ // first, read TFile meta
+ BlockReader brMeta = readerBCF.getMetaBlock(TFileMeta.BLOCK_NAME);
+ try {
+ tfileMeta = new TFileMeta(brMeta);
+ } finally {
+ brMeta.close();
+ }
+
+ comparator = tfileMeta.getComparator();
+ // Set begin and end locations.
+ begin = new Location(0, 0);
+ end = new Location(readerBCF.getBlockCount(), 0);
+ }
+
+ /**
+ * Close the reader. The state of the Reader object is undefined after close. Calling close() for multiple times has no effect.
+ */
+ public void close() throws IOException {
+ readerBCF.close();
+ }
+
+ /**
+ * Get the begin location of the TFile.
+ *
+ * @return If TFile is not empty, the location of the first key-value pair. Otherwise, it returns end().
+ */
+ Location begin() {
+ return begin;
+ }
+
+ /**
+ * Get the end location of the TFile.
+ *
+ * @return The location right after the last key-value pair in TFile.
+ */
+ Location end() {
+ return end;
+ }
+
+ /**
+ * Get the string representation of the comparator.
+ *
+ * @return If the TFile is not sorted by keys, an empty string will be returned. Otherwise, the actual comparator string that is provided during the TFile
+ * creation time will be returned.
+ */
+ public String getComparatorName() {
+ return tfileMeta.getComparatorString();
+ }
+
+ /**
+ * Is the TFile sorted?
+ *
+ * @return true if TFile is sorted.
+ */
+ public boolean isSorted() {
+ return tfileMeta.isSorted();
+ }
+
+ /**
+ * Get the number of key-value pair entries in TFile.
+ *
+ * @return the number of key-value pairs in TFile
+ */
+ public long getEntryCount() {
+ return tfileMeta.getRecordCount();
+ }
+
+ /**
+ * Lazily loading the TFile index.
+ *
+ * @throws IOException
+ */
+ synchronized void checkTFileDataIndex() throws IOException {
+ if (tfileIndex == null) {
+ BlockReader brIndex = readerBCF.getMetaBlock(TFileIndex.BLOCK_NAME);
+ try {
+ tfileIndex = new TFileIndex(readerBCF.getBlockCount(), brIndex, tfileMeta.getComparator());
+ } finally {
+ brIndex.close();
+ }
+ }
+ }
+
+ /**
+ * Get the first key in the TFile.
+ *
+ * @return The first key in the TFile.
- * @throws IOException
+ */
+ public RawComparable getFirstKey() throws IOException {
+ checkTFileDataIndex();
+ return tfileIndex.getFirstKey();
+ }
+
+ /**
+ * Get the last key in the TFile.
+ *
+ * @return The last key in the TFile.
- * @throws IOException
+ */
+ public RawComparable getLastKey() throws IOException {
+ checkTFileDataIndex();
+ return tfileIndex.getLastKey();
+ }
+
+ /**
+ * Get a Comparator object to compare Entries. It is useful when you want stores the entries in a collection (such as PriorityQueue) and perform sorting or
+ * comparison among entries based on the keys without copying out the key.
+ *
+ * @return An Entry Comparator..
+ */
+ public Comparator<Scanner.Entry> getEntryComparator() {
+ if (!isSorted()) {
+ throw new RuntimeException("Entries are not comparable for unsorted TFiles");
+ }
+
+ return new Comparator<Scanner.Entry>() {
+ /**
+ * Provide a customized comparator for Entries. This is useful if we have a collection of Entry objects. However, if the Entry objects come from
+ * different TFiles, users must ensure that those TFiles share the same RawComparator.
+ */
+ @Override
+ public int compare(Scanner.Entry o1, Scanner.Entry o2) {
+ return comparator.compare(o1.getKeyBuffer(), 0, o1.getKeyLength(), o2.getKeyBuffer(), 0, o2.getKeyLength());
+ }
+ };
+ }
+
+ /**
+ * Get an instance of the RawComparator that is constructed based on the string comparator representation.
+ *
+ * @return a Comparator that can compare RawComparable's.
+ */
+ public Comparator<RawComparable> getComparator() {
+ return comparator;
+ }
+
+ /**
+ * Stream access to a meta block.``
+ *
+ * @param name
+ * The name of the meta block.
+ * @return The input stream.
+ * @throws IOException
+ * on I/O error.
+ * @throws MetaBlockDoesNotExist
+ * If the meta block with the name does not exist.
+ */
+ public DataInputStream getMetaBlock(String name) throws IOException, MetaBlockDoesNotExist {
+ return readerBCF.getMetaBlock(name);
+ }
+
+ /**
+ * if greater is true then returns the beginning location of the block containing the key strictly greater than input key. if greater is false then returns
+ * the beginning location of the block greater than equal to the input key
+ *
+ * @param key
+ * the input key
+ * @param greater
+ * boolean flag
+ * @throws IOException
+ */
+ Location getBlockContainsKey(RawComparable key, boolean greater) throws IOException {
+ if (!isSorted()) {
+ throw new RuntimeException("Seeking in unsorted TFile");
+ }
+ checkTFileDataIndex();
+ int blkIndex = (greater) ? tfileIndex.upperBound(key) : tfileIndex.lowerBound(key);
+ if (blkIndex < 0)
+ return end;
+ return new Location(blkIndex, 0);
+ }
+
+ int compareKeys(byte[] a, int o1, int l1, byte[] b, int o2, int l2) {
+ if (!isSorted()) {
+ throw new RuntimeException("Cannot compare keys for unsorted TFiles.");
+ }
+ return comparator.compare(a, o1, l1, b, o2, l2);
+ }
+
+ int compareKeys(RawComparable a, RawComparable b) {
+ if (!isSorted()) {
+ throw new RuntimeException("Cannot compare keys for unsorted TFiles.");
+ }
+ return comparator.compare(a, b);
+ }
+
+ /**
+ * Get the location pointing to the beginning of the first key-value pair in a compressed block whose byte offset in the TFile is greater than or equal to
+ * the specified offset.
+ *
+ * @param offset
+ * the user supplied offset.
+ * @return the location to the corresponding entry; or end() if no such entry exists.
+ */
+ Location getLocationNear(long offset) {
+ int blockIndex = readerBCF.getBlockIndexNear(offset);
+ if (blockIndex == -1)
+ return end;
+ return new Location(blockIndex, 0);
+ }
+
+ /**
+ * Get a sample key that is within a block whose starting offset is greater than or equal to the specified offset.
+ *
+ * @param offset
+ * The file offset.
+ * @return the key that fits the requirement; or null if no such key exists (which could happen if the offset is close to the end of the TFile).
- * @throws IOException
+ */
+ public RawComparable getKeyNear(long offset) throws IOException {
+ int blockIndex = readerBCF.getBlockIndexNear(offset);
+ if (blockIndex == -1)
+ return null;
+ checkTFileDataIndex();
+ return new ByteArray(tfileIndex.getEntry(blockIndex).key);
+ }
+
+ /**
+ * Get a scanner than can scan the whole TFile.
+ *
+ * @return The scanner object. A valid Scanner is always returned even if the TFile is empty.
- * @throws IOException
+ */
+ public Scanner createScanner() throws IOException {
+ return new Scanner(this, begin, end);
+ }
+
+ /**
+ * Get a scanner that covers a portion of TFile based on byte offsets.
+ *
+ * @param offset
+ * The beginning byte offset in the TFile.
+ * @param length
+ * The length of the region.
+ * @return The actual coverage of the returned scanner tries to match the specified byte-region but always round up to the compression block boundaries. It
+ * is possible that the returned scanner contains zero key-value pairs even if length is positive.
- * @throws IOException
+ */
+ public Scanner createScanner(long offset, long length) throws IOException {
+ return new Scanner(this, offset, offset + length);
+ }
+
+ /**
+ * Get a scanner that covers a portion of TFile based on keys.
+ *
+ * @param beginKey
+ * Begin key of the scan (inclusive). If null, scan from the first key-value entry of the TFile.
+ * @param endKey
+ * End key of the scan (exclusive). If null, scan up to the last key-value entry of the TFile.
+ * @return The actual coverage of the returned scanner will cover all keys greater than or equal to the beginKey and less than the endKey.
- * @throws IOException
+ */
+ public Scanner createScanner(byte[] beginKey, byte[] endKey) throws IOException {
+ return createScanner((beginKey == null) ? null : new ByteArray(beginKey, 0, beginKey.length), (endKey == null) ? null : new ByteArray(endKey, 0,
+ endKey.length));
+ }
+
+ /**
+ * Get a scanner that covers a specific key range.
+ *
+ * @param beginKey
+ * Begin key of the scan (inclusive). If null, scan from the first key-value entry of the TFile.
+ * @param endKey
+ * End key of the scan (exclusive). If null, scan up to the last key-value entry of the TFile.
+ * @return The actual coverage of the returned scanner will cover all keys greater than or equal to the beginKey and less than the endKey.
- * @throws IOException
+ */
+ public Scanner createScanner(RawComparable beginKey, RawComparable endKey) throws IOException {
+ if ((beginKey != null) && (endKey != null) && (compareKeys(beginKey, endKey) >= 0)) {
+ return new Scanner(this, beginKey, beginKey);
+ }
+ return new Scanner(this, beginKey, endKey);
+ }
+
+ /**
+ * The TFile Scanner. The Scanner has an implicit cursor, which, upon creation, points to the first key-value pair in the scan range. If the scan range is
+ * empty, the cursor will point to the end of the scan range.
+ * <p>
+ * Use {@link Scanner#atEnd()} to test whether the cursor is at the end location of the scanner.
+ * <p>
+ * Use {@link Scanner#advance()} to move the cursor to the next key-value pair (or end if none exists). Use seekTo methods ( {@link Scanner#seekTo(byte[])}
+ * or {@link Scanner#seekTo(byte[], int, int)}) to seek to any arbitrary location in the covered range (including backward seeking). Use
+ * {@link Scanner#rewind()} to seek back to the beginning of the scanner. Use {@link Scanner#seekToEnd()} to seek to the end of the scanner.
+ * <p>
+ * Actual keys and values may be obtained through {@link Scanner.Entry} object, which is obtained through {@link Scanner#entry()}.
+ */
+ public static class Scanner implements Closeable {
+ // The underlying TFile reader.
+ final Reader reader;
+ // current block (null if reaching end)
+ private BlockReader blkReader;
+
+ Location beginLocation;
+ Location endLocation;
+ Location currentLocation;
+
+ // flag to ensure value is only examined once.
+ boolean valueChecked = false;
+ // reusable buffer for keys.
+ final byte[] keyBuffer;
+ // length of key, -1 means key is invalid.
+ int klen = -1;
+
+ static final int MAX_VAL_TRANSFER_BUF_SIZE = 128 * 1024;
+ BytesWritable valTransferBuffer;
+
+ DataInputBuffer keyDataInputStream;
+ ChunkDecoder valueBufferInputStream;
+ DataInputStream valueDataInputStream;
+ // vlen == -1 if unknown.
+ int vlen;
+
+ /**
+ * Constructor
+ *
+ * @param reader
+ * The TFile reader object.
+ * @param offBegin
+ * Begin byte-offset of the scan.
+ * @param offEnd
+ * End byte-offset of the scan.
+ * @throws IOException
+ *
+ * The offsets will be rounded to the beginning of a compressed block whose offset is greater than or equal to the specified offset.
+ */
+ protected Scanner(Reader reader, long offBegin, long offEnd) throws IOException {
+ this(reader, reader.getLocationNear(offBegin), reader.getLocationNear(offEnd));
+ }
+
+ /**
+ * Constructor
+ *
+ * @param reader
+ * The TFile reader object.
+ * @param begin
+ * Begin location of the scan.
+ * @param end
+ * End location of the scan.
+ * @throws IOException
+ */
+ Scanner(Reader reader, Location begin, Location end) throws IOException {
+ this.reader = reader;
+ // ensure the TFile index is loaded throughout the life of scanner.
+ reader.checkTFileDataIndex();
+ beginLocation = begin;
+ endLocation = end;
+
+ valTransferBuffer = new BytesWritable();
+ keyBuffer = new byte[MAX_KEY_SIZE];
+ keyDataInputStream = new DataInputBuffer();
+ valueBufferInputStream = new ChunkDecoder();
+ valueDataInputStream = new DataInputStream(valueBufferInputStream);
+
+ if (beginLocation.compareTo(endLocation) >= 0) {
+ currentLocation = new Location(endLocation);
+ } else {
+ currentLocation = new Location(0, 0);
+ initBlock(beginLocation.getBlockIndex());
+ inBlockAdvance(beginLocation.getRecordIndex());
+ }
+ }
+
+ /**
+ * Constructor
+ *
+ * @param reader
+ * The TFile reader object.
+ * @param beginKey
+ * Begin key of the scan. If null, scan from the first <K,V> entry of the TFile.
+ * @param endKey
+ * End key of the scan. If null, scan up to the last <K, V> entry of the TFile.
- * @throws IOException
+ */
+ protected Scanner(Reader reader, RawComparable beginKey, RawComparable endKey) throws IOException {
+ this(reader, (beginKey == null) ? reader.begin() : reader.getBlockContainsKey(beginKey, false), reader.end());
+ if (beginKey != null) {
+ inBlockAdvance(beginKey, false);
+ beginLocation.set(currentLocation);
+ }
+ if (endKey != null) {
+ seekTo(endKey, false);
+ endLocation.set(currentLocation);
+ seekTo(beginLocation);
+ }
+ }
+
+ /**
+ * Move the cursor to the first entry whose key is greater than or equal to the input key. Synonymous to seekTo(key, 0, key.length). The entry returned by
+ * the previous entry() call will be invalid.
+ *
+ * @param key
+ * The input key
+ * @return true if we find an equal key.
- * @throws IOException
+ */
+ public boolean seekTo(byte[] key) throws IOException {
+ return seekTo(key, 0, key.length);
+ }
+
+ /**
+ * Move the cursor to the first entry whose key is greater than or equal to the input key. The entry returned by the previous entry() call will be
+ * invalid.
+ *
+ * @param key
+ * The input key
+ * @param keyOffset
+ * offset in the key buffer.
+ * @param keyLen
+ * key buffer length.
+ * @return true if we find an equal key; false otherwise.
- * @throws IOException
+ */
+ public boolean seekTo(byte[] key, int keyOffset, int keyLen) throws IOException {
+ return seekTo(new ByteArray(key, keyOffset, keyLen), false);
+ }
+
+ private boolean seekTo(RawComparable key, boolean beyond) throws IOException {
+ Location l = reader.getBlockContainsKey(key, beyond);
+ if (l.compareTo(beginLocation) < 0) {
+ l = beginLocation;
+ } else if (l.compareTo(endLocation) >= 0) {
+ seekTo(endLocation);
+ return false;
+ }
+
+ // check if what we are seeking is in the later part of the current
+ // block.
+ if (atEnd() || (l.getBlockIndex() != currentLocation.getBlockIndex()) || (compareCursorKeyTo(key) >= 0)) {
+ // sorry, we must seek to a different location first.
+ seekTo(l);
+ }
+
+ return inBlockAdvance(key, beyond);
+ }
+
+ /**
+ * Move the cursor to the new location. The entry returned by the previous entry() call will be invalid.
+ *
+ * @param l
+ * new cursor location. It must fall between the begin and end location of the scanner.
+ * @throws IOException
+ */
+ private void seekTo(Location l) throws IOException {
+ if (l.compareTo(beginLocation) < 0) {
+ throw new IllegalArgumentException("Attempt to seek before the begin location.");
+ }
+
+ if (l.compareTo(endLocation) > 0) {
+ throw new IllegalArgumentException("Attempt to seek after the end location.");
+ }
+
+ if (l.compareTo(endLocation) == 0) {
+ parkCursorAtEnd();
+ return;
+ }
+
+ if (l.getBlockIndex() != currentLocation.getBlockIndex()) {
+ // going to a totally different block
+ initBlock(l.getBlockIndex());
+ } else {
+ if (valueChecked) {
+ // may temporarily go beyond the last record in the block (in which
+ // case the next if loop will always be true).
+ inBlockAdvance(1);
+ }
+ if (l.getRecordIndex() < currentLocation.getRecordIndex()) {
+ initBlock(l.getBlockIndex());
+ }
+ }
+
+ inBlockAdvance(l.getRecordIndex() - currentLocation.getRecordIndex());
+
+ return;
+ }
+
+ /**
+ * Rewind to the first entry in the scanner. The entry returned by the previous entry() call will be invalid.
- *
- * @throws IOException
+ */
+ public void rewind() throws IOException {
+ seekTo(beginLocation);
+ }
+
+ /**
+ * Seek to the end of the scanner. The entry returned by the previous entry() call will be invalid.
- *
- * @throws IOException
+ */
+ public void seekToEnd() throws IOException {
+ parkCursorAtEnd();
+ }
+
+ /**
+ * Move the cursor to the first entry whose key is greater than or equal to the input key. Synonymous to lowerBound(key, 0, key.length). The entry
+ * returned by the previous entry() call will be invalid.
+ *
+ * @param key
+ * The input key
- * @throws IOException
+ */
+ public void lowerBound(byte[] key) throws IOException {
+ lowerBound(key, 0, key.length);
+ }
+
+ /**
+ * Move the cursor to the first entry whose key is greater than or equal to the input key. The entry returned by the previous entry() call will be
+ * invalid.
+ *
+ * @param key
+ * The input key
+ * @param keyOffset
+ * offset in the key buffer.
+ * @param keyLen
+ * key buffer length.
- * @throws IOException
+ */
+ public void lowerBound(byte[] key, int keyOffset, int keyLen) throws IOException {
+ seekTo(new ByteArray(key, keyOffset, keyLen), false);
+ }
+
+ /**
+ * Move the cursor to the first entry whose key is strictly greater than the input key. Synonymous to upperBound(key, 0, key.length). The entry returned
+ * by the previous entry() call will be invalid.
+ *
+ * @param key
+ * The input key
- * @throws IOException
+ */
+ public void upperBound(byte[] key) throws IOException {
+ upperBound(key, 0, key.length);
+ }
+
+ /**
+ * Move the cursor to the first entry whose key is strictly greater than the input key. The entry returned by the previous entry() call will be invalid.
+ *
+ * @param key
+ * The input key
+ * @param keyOffset
+ * offset in the key buffer.
+ * @param keyLen
+ * key buffer length.
- * @throws IOException
+ */
+ public void upperBound(byte[] key, int keyOffset, int keyLen) throws IOException {
+ seekTo(new ByteArray(key, keyOffset, keyLen), true);
+ }
+
+ /**
+ * Move the cursor to the next key-value pair. The entry returned by the previous entry() call will be invalid.
+ *
+ * @return true if the cursor successfully moves. False when cursor is already at the end location and cannot be advanced.
- * @throws IOException
+ */
+ public boolean advance() throws IOException {
+ if (atEnd()) {
+ return false;
+ }
+
+ int curBid = currentLocation.getBlockIndex();
+ long curRid = currentLocation.getRecordIndex();
+ long entriesInBlock = reader.getBlockEntryCount(curBid);
+ if (curRid + 1 >= entriesInBlock) {
+ if (endLocation.compareTo(curBid + 1, 0) <= 0) {
+ // last entry in TFile.
+ parkCursorAtEnd();
+ } else {
+ // last entry in Block.
+ initBlock(curBid + 1);
+ }
+ } else {
+ inBlockAdvance(1);
+ }
+ return true;
+ }
+
+ /**
+ * Load a compressed block for reading. Expecting blockIndex is valid.
+ *
+ * @throws IOException
+ */
+ private void initBlock(int blockIndex) throws IOException {
+ klen = -1;
+ if (blkReader != null) {
+ try {
+ blkReader.close();
+ } finally {
+ blkReader = null;
+ }
+ }
+ blkReader = reader.getBlockReader(blockIndex);
+ currentLocation.set(blockIndex, 0);
+ }
+
+ private void parkCursorAtEnd() throws IOException {
+ klen = -1;
+ currentLocation.set(endLocation);
+ if (blkReader != null) {
+ try {
+ blkReader.close();
+ } finally {
+ blkReader = null;
+ }
+ }
+ }
+
+ /**
+ * Close the scanner. Release all resources. The behavior of using the scanner after calling close is not defined. The entry returned by the previous
+ * entry() call will be invalid.
+ */
+ public void close() throws IOException {
+ parkCursorAtEnd();
+ }
+
+ /**
+ * Is cursor at the end location?
+ *
+ * @return true if the cursor is at the end location.
+ */
+ public boolean atEnd() {
+ return (currentLocation.compareTo(endLocation) >= 0);
+ }
+
+ /**
+ * check whether we have already successfully obtained the key. It also initializes the valueInputStream.
+ */
+ void checkKey() throws IOException {
+ if (klen >= 0)
+ return;
+ if (atEnd()) {
+ throw new EOFException("No key-value to read");
+ }
+ klen = -1;
+ vlen = -1;
+ valueChecked = false;
+
+ klen = Utils.readVInt(blkReader);
+ blkReader.readFully(keyBuffer, 0, klen);
+ valueBufferInputStream.reset(blkReader);
+ if (valueBufferInputStream.isLastChunk()) {
+ vlen = valueBufferInputStream.getRemain();
+ }
+ }
+
+ /**
+ * Get an entry to access the key and value.
+ *
+ * @return The Entry object to access the key and value.
- * @throws IOException
+ */
+ public Entry entry() throws IOException {
+ checkKey();
+ return new Entry();
+ }
+
+ /**
+ * Internal API. Comparing the key at cursor to user-specified key.
+ *
+ * @param other
+ * user-specified key.
+ * @return negative if key at cursor is smaller than user key; 0 if equal; and positive if key at cursor greater than user key.
+ * @throws IOException
+ */
+ int compareCursorKeyTo(RawComparable other) throws IOException {
+ checkKey();
+ return reader.compareKeys(keyBuffer, 0, klen, other.buffer(), other.offset(), other.size());
+ }
+
+ /**
+ * Entry to a <Key, Value> pair.
+ */
+ public class Entry implements Comparable<RawComparable> {
+ /**
+ * Get the length of the key.
+ *
+ * @return the length of the key.
+ */
+ public int getKeyLength() {
+ return klen;
+ }
+
+ byte[] getKeyBuffer() {
+ return keyBuffer;
+ }
+
+ /**
+ * Copy the key and value in one shot into BytesWritables. This is equivalent to getKey(key); getValue(value);
+ *
+ * @param key
+ * BytesWritable to hold key.
+ * @param value
+ * BytesWritable to hold value
- * @throws IOException
+ */
+ public void get(BytesWritable key, BytesWritable value) throws IOException {
+ getKey(key);
+ getValue(value);
+ }
+
+ /**
+ * Copy the key into BytesWritable. The input BytesWritable will be automatically resized to the actual key size.
+ *
+ * @param key
+ * BytesWritable to hold the key.
- * @throws IOException
+ */
+ public int getKey(BytesWritable key) throws IOException {
+ key.setSize(getKeyLength());
+ getKey(key.getBytes());
+ return key.getLength();
+ }
+
+ /**
+ * Copy the value into BytesWritable. The input BytesWritable will be automatically resized to the actual value size. The implementation directly uses
+ * the buffer inside BytesWritable for storing the value. The call does not require the value length to be known.
- *
- * @param value
- * @throws IOException
+ */
+ public long getValue(BytesWritable value) throws IOException {
+ DataInputStream dis = getValueStream();
+ int size = 0;
+ try {
+ int remain;
+ while ((remain = valueBufferInputStream.getRemain()) > 0) {
+ value.setSize(size + remain);
+ dis.readFully(value.getBytes(), size, remain);
+ size += remain;
+ }
+ return value.getLength();
+ } finally {
+ dis.close();
+ }
+ }
+
+ /**
+ * Writing the key to the output stream. This method avoids copying key buffer from Scanner into user buffer, then writing to the output stream.
+ *
+ * @param out
+ * The output stream
+ * @return the length of the key.
- * @throws IOException
+ */
+ public int writeKey(OutputStream out) throws IOException {
+ out.write(keyBuffer, 0, klen);
+ return klen;
+ }
+
+ /**
+ * Writing the value to the output stream. This method avoids copying value data from Scanner into user buffer, then writing to the output stream. It
+ * does not require the value length to be known.
+ *
+ * @param out
+ * The output stream
+ * @return the length of the value
- * @throws IOException
+ */
+ public long writeValue(OutputStream out) throws IOException {
+ DataInputStream dis = getValueStream();
+ long size = 0;
+ try {
+ int chunkSize;
+ while ((chunkSize = valueBufferInputStream.getRemain()) > 0) {
+ chunkSize = Math.min(chunkSize, MAX_VAL_TRANSFER_BUF_SIZE);
+ valTransferBuffer.setSize(chunkSize);
+ dis.readFully(valTransferBuffer.getBytes(), 0, chunkSize);
+ out.write(valTransferBuffer.getBytes(), 0, chunkSize);
+ size += chunkSize;
+ }
+ return size;
+ } finally {
+ dis.close();
+ }
+ }
+
+ /**
+ * Copy the key into user supplied buffer.
+ *
+ * @param buf
+ * The buffer supplied by user. The length of the buffer must not be shorter than the key length.
+ * @return The length of the key.
- *
- * @throws IOException
+ */
+ public int getKey(byte[] buf) throws IOException {
+ return getKey(buf, 0);
+ }
+
+ /**
+ * Copy the key into user supplied buffer.
+ *
+ * @param buf
+ * The buffer supplied by user.
+ * @param offset
+ * The starting offset of the user buffer where we should copy the key into. Requiring the key-length + offset no greater than the buffer
+ * length.
+ * @return The length of the key.
- * @throws IOException
+ */
+ public int getKey(byte[] buf, int offset) throws IOException {
+ if ((offset | (buf.length - offset - klen)) < 0) {
+ throw new IndexOutOfBoundsException("Bufer not enough to store the key");
+ }
+ System.arraycopy(keyBuffer, 0, buf, offset, klen);
+ return klen;
+ }
+
+ /**
+ * Streaming access to the key. Useful for desrializing the key into user objects.
+ *
+ * @return The input stream.
+ */
+ public DataInputStream getKeyStream() {
+ keyDataInputStream.reset(keyBuffer, klen);
+ return keyDataInputStream;
+ }
+
+ /**
+ * Get the length of the value. isValueLengthKnown() must be tested true.
+ *
+ * @return the length of the value.
+ */
+ public int getValueLength() {
+ if (vlen >= 0) {
+ return vlen;
+ }
+
+ throw new RuntimeException("Value length unknown.");
+ }
+
+ /**
+ * Copy value into user-supplied buffer. User supplied buffer must be large enough to hold the whole value. The value part of the key-value pair pointed
+ * by the current cursor is not cached and can only be examined once. Calling any of the following functions more than once without moving the cursor
+ * will result in exception: {@link #getValue(byte[])}, {@link #getValue(byte[], int)}, {@link #getValueStream}.
+ *
+ * @return the length of the value. Does not require isValueLengthKnown() to be true.
- * @throws IOException
+ *
+ */
+ public int getValue(byte[] buf) throws IOException {
+ return getValue(buf, 0);
+ }
+
+ /**
+ * Copy value into user-supplied buffer. User supplied buffer must be large enough to hold the whole value (starting from the offset). The value part of
+ * the key-value pair pointed by the current cursor is not cached and can only be examined once. Calling any of the following functions more than once
+ * without moving the cursor will result in exception: {@link #getValue(byte[])}, {@link #getValue(byte[], int)}, {@link #getValueStream}.
+ *
+ * @return the length of the value. Does not require isValueLengthKnown() to be true.
- * @throws IOException
+ */
+ public int getValue(byte[] buf, int offset) throws IOException {
+ DataInputStream dis = getValueStream();
+ try {
+ if (isValueLengthKnown()) {
+ if ((offset | (buf.length - offset - vlen)) < 0) {
+ throw new IndexOutOfBoundsException("Buffer too small to hold value");
+ }
+ dis.readFully(buf, offset, vlen);
+ return vlen;
+ }
+
+ int nextOffset = offset;
+ while (nextOffset < buf.length) {
+ int n = dis.read(buf, nextOffset, buf.length - nextOffset);
+ if (n < 0) {
+ break;
+ }
+ nextOffset += n;
+ }
+ if (dis.read() >= 0) {
+ // attempt to read one more byte to determine whether we reached
+ // the
+ // end or not.
+ throw new IndexOutOfBoundsException("Buffer too small to hold value");
+ }
+ return nextOffset - offset;
+ } finally {
+ dis.close();
+ }
+ }
+
+ /**
+ * Stream access to value. The value part of the key-value pair pointed by the current cursor is not cached and can only be examined once. Calling any
+ * of the following functions more than once without moving the cursor will result in exception: {@link #getValue(byte[])},
+ * {@link #getValue(byte[], int)}, {@link #getValueStream}.
+ *
+ * @return The input stream for reading the value.
- * @throws IOException
+ */
+ public DataInputStream getValueStream() throws IOException {
+ if (valueChecked == true) {
+ throw new IllegalStateException("Attempt to examine value multiple times.");
+ }
+ valueChecked = true;
+ return valueDataInputStream;
+ }
+
+ /**
+ * Check whether it is safe to call getValueLength().
+ *
+ * @return true if value length is known before hand. Values less than the chunk size will always have their lengths known before hand. Values that are
+ * written out as a whole (with advertised length up-front) will always have their lengths known in read.
+ */
+ public boolean isValueLengthKnown() {
+ return (vlen >= 0);
+ }
+
+ /**
+ * Compare the entry key to another key. Synonymous to compareTo(key, 0, key.length).
+ *
+ * @param buf
+ * The key buffer.
+ * @return comparison result between the entry key with the input key.
+ */
+ public int compareTo(byte[] buf) {
+ return compareTo(buf, 0, buf.length);
+ }
+
+ /**
+ * Compare the entry key to another key. Synonymous to compareTo(new ByteArray(buf, offset, length)
+ *
+ * @param buf
+ * The key buffer
+ * @param offset
+ * offset into the key buffer.
+ * @param length
+ * the length of the key.
+ * @return comparison result between the entry key with the input key.
+ */
+ public int compareTo(byte[] buf, int offset, int length) {
+ return compareTo(new ByteArray(buf, offset, length));
+ }
+
+ /**
+ * Compare an entry with a RawComparable object. This is useful when Entries are stored in a collection, and we want to compare a user supplied key.
+ */
+ @Override
+ public int compareTo(RawComparable key) {
+ return reader.compareKeys(keyBuffer, 0, getKeyLength(), key.buffer(), key.offset(), key.size());
+ }
+
+ /**
+ * Compare whether this and other points to the same key value.
+ */
+ @Override
+ public boolean equals(Object other) {
+ if (this == other)
+ return true;
+ if (!(other instanceof Entry))
+ return false;
+ return ((Entry) other).compareTo(keyBuffer, 0, getKeyLength()) == 0;
+ }
+
+ @Override
+ public int hashCode() {
+ return WritableComparator.hashBytes(keyBuffer, getKeyLength());
+ }
+ }
+
+ /**
+ * Advance cursor by n positions within the block.
+ *
+ * @param n
+ * Number of key-value pairs to skip in block.
+ * @throws IOException
+ */
+ private void inBlockAdvance(long n) throws IOException {
+ for (long i = 0; i < n; ++i) {
+ checkKey();
+ if (!valueBufferInputStream.isClosed()) {
+ valueBufferInputStream.close();
+ }
+ klen = -1;
+ currentLocation.incRecordIndex();
+ }
+ }
+
+ /**
+ * Advance cursor in block until we find a key that is greater than or equal to the input key.
+ *
+ * @param key
+ * Key to compare.
+ * @param greater
+ * advance until we find a key greater than the input key.
+ * @return true if we find a equal key.
+ * @throws IOException
+ */
+ private boolean inBlockAdvance(RawComparable key, boolean greater) throws IOException {
+ int curBid = currentLocation.getBlockIndex();
+ long entryInBlock = reader.getBlockEntryCount(curBid);
+ if (curBid == endLocation.getBlockIndex()) {
+ entryInBlock = endLocation.getRecordIndex();
+ }
+
+ while (currentLocation.getRecordIndex() < entryInBlock) {
+ int cmp = compareCursorKeyTo(key);
+ if (cmp > 0)
+ return false;
+ if (cmp == 0 && !greater)
+ return true;
+ if (!valueBufferInputStream.isClosed()) {
+ valueBufferInputStream.close();
+ }
+ klen = -1;
+ currentLocation.incRecordIndex();
+ }
+
+ throw new RuntimeException("Cannot find matching key in block.");
+ }
+ }
+
+ long getBlockEntryCount(int curBid) {
+ return tfileIndex.getEntry(curBid).entries();
+ }
+
+ BlockReader getBlockReader(int blockIndex) throws IOException {
+ return readerBCF.getDataBlock(blockIndex);
+ }
+ }
+
+ /**
+ * Data structure representing "TFile.meta" meta block.
+ */
+ static final class TFileMeta {
+ final static String BLOCK_NAME = "TFile.meta";
+ final Version version;
+ private long recordCount;
+ private final String strComparator;
+ private final BytesComparator comparator;
+
+ // ctor for writes
+ public TFileMeta(String comparator) {
+ // set fileVersion to API version when we create it.
+ version = TFile.API_VERSION;
+ recordCount = 0;
+ strComparator = (comparator == null) ? "" : comparator;
+ this.comparator = makeComparator(strComparator);
+ }
+
+ // ctor for reads
+ public TFileMeta(DataInput in) throws IOException {
+ version = new Version(in);
+ if (!version.compatibleWith(TFile.API_VERSION)) {
+ throw new RuntimeException("Incompatible TFile fileVersion.");
+ }
+ recordCount = Utils.readVLong(in);
+ strComparator = Utils.readString(in);
+ comparator = makeComparator(strComparator);
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ static BytesComparator makeComparator(String comparator) {
+ if (comparator.length() == 0) {
+ // unsorted keys
+ return null;
+ }
+ if (comparator.equals(COMPARATOR_MEMCMP)) {
+ // default comparator
+ return new BytesComparator(new MemcmpRawComparator());
+ } else if (comparator.startsWith(COMPARATOR_JCLASS)) {
+ String compClassName = comparator.substring(COMPARATOR_JCLASS.length()).trim();
+ try {
+ Class compClass = Class.forName(compClassName);
+ // use its default ctor to create an instance
+ return new BytesComparator((RawComparator<Object>) compClass.newInstance());
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Failed to instantiate comparator: " + comparator + "(" + e.toString() + ")");
+ }
+ } else {
+ throw new IllegalArgumentException("Unsupported comparator: " + comparator);
+ }
+ }
+
+ public void write(DataOutput out) throws IOException {
+ TFile.API_VERSION.write(out);
+ Utils.writeVLong(out, recordCount);
+ Utils.writeString(out, strComparator);
+ }
+
+ public long getRecordCount() {
+ return recordCount;
+ }
+
+ public void incRecordCount() {
+ ++recordCount;
+ }
+
+ public boolean isSorted() {
+ return !strComparator.equals("");
+ }
+
+ public String getComparatorString() {
+ return strComparator;
+ }
+
+ public BytesComparator getComparator() {
+ return comparator;
+ }
+
+ public Version getVersion() {
+ return version;
+ }
+ } // END: class MetaTFileMeta
+
+ /**
+ * Data structure representing "TFile.index" meta block.
+ */
+ static class TFileIndex {
+ final static String BLOCK_NAME = "TFile.index";
+ private ByteArray firstKey;
+ private final ArrayList<TFileIndexEntry> index;
+ private final BytesComparator comparator;
+
+ /**
+ * For reading from file.
- *
- * @throws IOException
+ */
+ public TFileIndex(int entryCount, DataInput in, BytesComparator comparator) throws IOException {
+ index = new ArrayList<TFileIndexEntry>(entryCount);
+ int size = Utils.readVInt(in); // size for the first key entry.
+ if (size > 0) {
+ byte[] buffer = new byte[size];
+ in.readFully(buffer);
+ DataInputStream firstKeyInputStream = new DataInputStream(new ByteArrayInputStream(buffer, 0, size));
+
+ int firstKeyLength = Utils.readVInt(firstKeyInputStream);
+ firstKey = new ByteArray(new byte[firstKeyLength]);
+ firstKeyInputStream.readFully(firstKey.buffer());
+
+ for (int i = 0; i < entryCount; i++) {
+ size = Utils.readVInt(in);
+ if (buffer.length < size) {
+ buffer = new byte[size];
+ }
+ in.readFully(buffer, 0, size);
+ TFileIndexEntry idx = new TFileIndexEntry(new DataInputStream(new ByteArrayInputStream(buffer, 0, size)));
+ index.add(idx);
+ }
+ } else {
+ if (entryCount != 0) {
+ throw new RuntimeException("Internal error");
+ }
+ }
+ this.comparator = comparator;
+ }
+
+ /**
+ * @param key
+ * input key.
+ * @return the ID of the first block that contains key >= input key. Or -1 if no such block exists.
+ */
+ public int lowerBound(RawComparable key) {
+ if (comparator == null) {
+ throw new RuntimeException("Cannot search in unsorted TFile");
+ }
+
+ if (firstKey == null) {
+ return -1; // not found
+ }
+
+ int ret = Utils.lowerBound(index, key, comparator);
+ if (ret == index.size()) {
+ return -1;
+ }
+ return ret;
+ }
+
+ public int upperBound(RawComparable key) {
+ if (comparator == null) {
+ throw new RuntimeException("Cannot search in unsorted TFile");
+ }
+
+ if (firstKey == null) {
+ return -1; // not found
+ }
+
+ int ret = Utils.upperBound(index, key, comparator);
+ if (ret == index.size()) {
+ return -1;
+ }
+ return ret;
+ }
+
+ /**
+ * For writing to file.
+ */
+ public TFileIndex(BytesComparator comparator) {
+ index = new ArrayList<TFileIndexEntry>();
+ this.comparator = comparator;
+ }
+
+ public RawComparable getFirstKey() {
+ return firstKey;
+ }
+
+ public void setFirstKey(byte[] key, int offset, int length) {
+ firstKey = new ByteArray(new byte[length]);
+ System.arraycopy(key, offset, firstKey.buffer(), 0, length);
+ }
+
+ public RawComparable getLastKey() {
+ if (index.size() == 0) {
+ return null;
+ }
+ return new ByteArray(index.get(index.size() - 1).buffer());
+ }
+
+ public void addEntry(TFileIndexEntry keyEntry) {
+ index.add(keyEntry);
+ }
+
+ public TFileIndexEntry getEntry(int bid) {
+ return index.get(bid);
+ }
+
+ public void write(DataOutput out) throws IOException {
+ if (firstKey == null) {
+ Utils.writeVInt(out, 0);
+ return;
+ }
+
+ DataOutputBuffer dob = new DataOutputBuffer();
+ Utils.writeVInt(dob, firstKey.size());
+ dob.write(firstKey.buffer());
+ Utils.writeVInt(out, dob.size());
+ out.write(dob.getData(), 0, dob.getLength());
+
+ for (TFileIndexEntry entry : index) {
+ dob.reset();
+ entry.write(dob);
+ Utils.writeVInt(out, dob.getLength());
+ out.write(dob.getData(), 0, dob.getLength());
+ }
+ }
+ }
+
+ /**
+ * TFile Data Index entry. We should try to make the memory footprint of each index entry as small as possible.
+ */
+ static final class TFileIndexEntry implements RawComparable {
+ final byte[] key;
+ // count of <key, value> entries in the block.
+ final long kvEntries;
+
+ public TFileIndexEntry(DataInput in) throws IOException {
+ int len = Utils.readVInt(in);
+ key = new byte[len];
+ in.readFully(key, 0, len);
+ kvEntries = Utils.readVLong(in);
+ }
+
+ // default entry, without any padding
+ public TFileIndexEntry(byte[] newkey, int offset, int len, long entries) {
+ key = new byte[len];
+ System.arraycopy(newkey, offset, key, 0, len);
+ this.kvEntries = entries;
+ }
+
+ @Override
+ public byte[] buffer() {
+ return key;
+ }
+
+ @Override
+ public int offset() {
+ return 0;
+ }
+
+ @Override
+ public int size() {
+ return key.length;
+ }
+
+ long entries() {
+ return kvEntries;
+ }
+
+ public void write(DataOutput out) throws IOException {
+ Utils.writeVInt(out, key.length);
+ out.write(key, 0, key.length);
+ Utils.writeVLong(out, kvEntries);
+ }
+ }
+
+ /**
+ * Dumping the TFile information.
+ *
+ * @param args
+ * A list of TFile paths.
+ */
+ public static void main(String[] args) {
+ System.out.printf("TFile Dumper (TFile %s, BCFile %s)%n", TFile.API_VERSION.toString(), BCFile.API_VERSION.toString());
+ if (args.length == 0) {
+ System.out.println("Usage: java ... org.apache.hadoop.io.file.tfile.TFile tfile-path [tfile-path ...]");
+ System.exit(0);
+ }
+ Configuration conf = new Configuration();
+
+ for (String file : args) {
+ System.out.println("===" + file + "===");
+ try {
+ TFileDumper.dumpInfo(file, System.out, conf);
+ } catch (IOException e) {
+ e.printStackTrace(System.err);
+ }
+ }
+ }
+}
[63/64] [abbrv] git commit: Merge branch '1.5.2-SNAPSHOT' into
1.6.0-SNAPSHOT
Posted by ct...@apache.org.
Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Conflicts:
core/src/main/java/org/apache/accumulo/core/Constants.java
core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
core/src/main/java/org/apache/accumulo/core/data/ColumnUpdate.java
core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Chunk.java
core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFile.java
core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFileDumper.java
core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModule.java
core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
core/src/test/java/org/apache/accumulo/core/util/shell/command/FormatterCommandTest.java
examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
server/base/src/main/java/org/apache/accumulo/server/util/TServerUtils.java
server/src/main/java/org/apache/accumulo/server/metanalysis/LogFileOutputFormat.java
server/src/main/java/org/apache/accumulo/server/metanalysis/PrintEvents.java
server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java
test/src/main/java/org/apache/accumulo/test/GetMasterStats.java
test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/716ea0ee
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/716ea0ee
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/716ea0ee
Branch: refs/heads/master
Commit: 716ea0ee8b26bf504d0cf9e90fc1d3d8579bc50a
Parents: 3934ea6 9261338
Author: Christopher Tubbs <ct...@apache.org>
Authored: Wed Apr 9 13:36:22 2014 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Wed Apr 9 13:36:22 2014 -0400
----------------------------------------------------------------------
.../core/client/ClientSideIteratorScanner.java | 2 -
.../accumulo/core/client/ConditionalWriter.java | 2 -
.../core/client/ConditionalWriterConfig.java | 2 -
.../apache/accumulo/core/client/Connector.java | 2 -
.../accumulo/core/client/IteratorSetting.java | 4 -
.../accumulo/core/client/RowIterator.java | 4 -
.../accumulo/core/client/ScannerBase.java | 1 -
.../core/client/admin/ActiveCompaction.java | 1 -
.../core/client/admin/InstanceOperations.java | 12 --
.../core/client/admin/TableOperations.java | 30 ----
.../core/client/admin/TableOperationsImpl.java | 4 -
.../core/client/impl/ThriftTransportPool.java | 16 +-
.../core/client/mapred/AbstractInputFormat.java | 2 -
.../client/mapred/AccumuloOutputFormat.java | 2 -
.../client/mapreduce/AbstractInputFormat.java | 3 +-
.../client/mapreduce/AccumuloOutputFormat.java | 2 -
.../core/client/mapreduce/InputTableConfig.java | 3 -
.../mapreduce/lib/util/ConfiguratorBase.java | 2 -
.../core/client/mock/MockBatchDeleter.java | 4 -
.../apache/accumulo/core/data/Condition.java | 7 -
.../java/org/apache/accumulo/core/data/Key.java | 7 +-
.../org/apache/accumulo/core/data/Range.java | 17 +-
.../accumulo/core/file/rfile/BlockIndex.java | 5 -
.../accumulo/core/file/rfile/bcfile/BCFile.java | 15 +-
.../core/file/rfile/bcfile/ByteArray.java | 2 -
.../accumulo/core/file/rfile/bcfile/Utils.java | 11 --
.../core/iterators/TypedValueCombiner.java | 6 -
.../core/iterators/ValueFormatException.java | 6 -
.../core/iterators/system/MapFileIterator.java | 8 +-
.../core/iterators/user/GrepIterator.java | 3 -
.../iterators/user/IntersectingIterator.java | 10 --
.../accumulo/core/iterators/user/RowFilter.java | 1 -
.../iterators/user/TransformingIterator.java | 164 +++++++++----------
.../core/iterators/user/VersioningIterator.java | 3 -
.../accumulo/core/security/SecurityUtil.java | 1 -
.../security/crypto/CryptoModuleFactory.java | 1 -
.../security/crypto/CryptoModuleParameters.java | 6 -
.../core/client/impl/ScannerOptionsTest.java | 2 -
.../client/lexicoder/ReverseLexicoderTest.java | 2 -
.../client/mapred/AccumuloInputFormatTest.java | 4 -
.../mapreduce/AccumuloInputFormatTest.java | 4 -
.../core/client/mock/MockNamespacesTest.java | 8 -
.../core/security/VisibilityConstraintTest.java | 3 -
.../simple/client/RandomBatchScanner.java | 5 -
.../simple/client/RandomBatchWriter.java | 4 -
.../simple/client/SequentialBatchWriter.java | 5 -
.../simple/client/TraceDumpExample.java | 9 +-
.../examples/simple/dirlist/QueryUtil.java | 3 -
.../examples/simple/mapreduce/NGramIngest.java | 3 -
.../examples/simple/mapreduce/TableToFile.java | 1 -
.../accumulo/examples/simple/shard/Query.java | 3 -
.../minicluster/MiniAccumuloRunner.java | 2 -
.../accumulo/proxy/TestProxyReadWrite.java | 10 --
.../accumulo/server/conf/ConfigSanityCheck.java | 3 -
.../server/master/balancer/TabletBalancer.java | 2 -
.../server/master/state/TabletStateStore.java | 7 -
.../server/metrics/AbstractMetricsImpl.java | 4 -
.../server/security/handler/ZKAuthorizor.java | 4 -
.../server/security/handler/ZKPermHandler.java | 4 -
.../accumulo/server/util/LoginProperties.java | 3 -
.../accumulo/server/util/RestoreZookeeper.java | 4 -
.../accumulo/server/util/TableDiskUsage.java | 3 -
.../accumulo/server/util/TabletServerLocks.java | 3 -
.../org/apache/accumulo/tserver/MemValue.java | 4 +-
.../apache/accumulo/tserver/TabletServer.java | 16 +-
.../tserver/compaction/CompactionStrategy.java | 4 -
.../accumulo/tserver/logger/LogReader.java | 1 -
.../start/classloader/AccumuloClassLoader.java | 3 -
.../start/classloader/vfs/ContextManager.java | 2 -
.../vfs/PostDelegatingVFSClassLoader.java | 7 +-
.../vfs/providers/HdfsFileSystem.java | 5 -
.../accumulo/test/NativeMapPerformanceTest.java | 3 -
.../accumulo/test/NativeMapStressTest.java | 3 -
.../test/continuous/ContinuousMoru.java | 1 -
.../test/continuous/ContinuousVerify.java | 1 -
.../test/functional/CacheTestClean.java | 3 -
.../accumulo/test/randomwalk/Framework.java | 3 -
.../apache/accumulo/test/randomwalk/Node.java | 1 -
.../randomwalk/concurrent/CheckBalance.java | 5 +-
79 files changed, 124 insertions(+), 414 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/ConditionalWriter.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/ConditionalWriter.java
index 5fdccf0,0000000..95f73bb
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/ConditionalWriter.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ConditionalWriter.java
@@@ -1,141 -1,0 +1,139 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.core.client;
+
+import java.util.Iterator;
+
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.data.ConditionalMutation;
+
+/**
+ * ConditionalWriter provides the ability to do efficient, atomic read-modify-write operations on rows. These operations are performed on the tablet server
+ * while a row lock is held.
+ *
+ * @since 1.6.0
+ */
+public interface ConditionalWriter {
+ class Result {
+
+ private Status status;
+ private ConditionalMutation mutation;
+ private String server;
+ private Exception exception;
+
+ public Result(Status s, ConditionalMutation m, String server) {
+ this.status = s;
+ this.mutation = m;
+ this.server = server;
+ }
+
+ public Result(Exception e, ConditionalMutation cm, String server) {
+ this.exception = e;
+ this.mutation = cm;
+ this.server = server;
+ }
+
+ /**
+ * If this method throws an exception, then its possible the mutation is still being actively processed. Therefore if code chooses to continue after seeing
+ * an exception it should take this into consideration.
+ *
+ * @return status of a conditional mutation
- * @throws AccumuloException
- * @throws AccumuloSecurityException
+ */
+
+ public Status getStatus() throws AccumuloException, AccumuloSecurityException {
+ if (status == null) {
+ if (exception instanceof AccumuloException)
+ throw new AccumuloException(exception);
+ if (exception instanceof AccumuloSecurityException) {
+ AccumuloSecurityException ase = (AccumuloSecurityException) exception;
+ throw new AccumuloSecurityException(ase.getUser(), SecurityErrorCode.valueOf(ase.getSecurityErrorCode().name()), ase.getTableInfo(), ase);
+ }
+ else
+ throw new AccumuloException(exception);
+ }
+
+ return status;
+ }
+
+ /**
+ *
+ * @return A copy of the mutation previously submitted by a user. The mutation will reference the same data, but the object may be different.
+ */
+ public ConditionalMutation getMutation() {
+ return mutation;
+ }
+
+ /**
+ *
+ * @return The server this mutation was sent to. Returns null if was not sent to a server.
+ */
+ public String getTabletServer() {
+ return server;
+ }
+ }
+
+ public static enum Status {
+ /**
+ * conditions were met and mutation was written
+ */
+ ACCEPTED,
+ /**
+ * conditions were not met and mutation was not written
+ */
+ REJECTED,
+ /**
+ * mutation violated a constraint and was not written
+ */
+ VIOLATED,
+ /**
+ * error occurred after mutation was sent to server, its unknown if the mutation was written. Although the status of the mutation is unknown, Accumulo
+ * guarantees the mutation will not be written at a later point in time.
+ */
+ UNKNOWN,
+ /**
+ * A condition contained a column visibility that could never be seen
+ */
+ INVISIBLE_VISIBILITY,
+
+ }
+
+ /**
+ * This method returns one result for each mutation passed to it. This method is thread safe. Multiple threads can safely use a single conditional writer.
+ * Sharing a conditional writer between multiple threads may result in batching of request to tablet servers.
+ *
+ * @param mutations
+ * @return Result for each mutation submitted. The mutations may still be processing in the background when this method returns, if so the iterator will
+ * block.
+ */
+ Iterator<Result> write(Iterator<ConditionalMutation> mutations);
+
+ /**
+ * This method has the same thread safety guarantees as @link {@link #write(Iterator)}
+ *
+ *
+ * @param mutation
+ * @return Result for the submitted mutation
+ */
+
+ Result write(ConditionalMutation mutation);
+
+ /**
+ * release any resources (like threads pools) used by conditional writer
+ */
+ void close();
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/ConditionalWriterConfig.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/ConditionalWriterConfig.java
index f2a91ea,0000000..a220e62
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/ConditionalWriterConfig.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ConditionalWriterConfig.java
@@@ -1,118 -1,0 +1,116 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.ArgumentChecker;
+
+/**
+ *
+ * @since 1.6.0
+ */
+public class ConditionalWriterConfig {
+
+ private static final Long DEFAULT_TIMEOUT = Long.MAX_VALUE;
+ private Long timeout = null;
+
+ private static final Integer DEFAULT_MAX_WRITE_THREADS = 3;
+ private Integer maxWriteThreads = null;
+
+ private Authorizations auths = Authorizations.EMPTY;
+
+ /**
+ * A set of authorization labels that will be checked against the column visibility of each key in order to filter data. The authorizations passed in must be
+ * a subset of the accumulo user's set of authorizations. If the accumulo user has authorizations (A1, A2) and authorizations (A2, A3) are passed, then an
+ * exception will be thrown.
+ *
+ * <p>
+ * Any condition that is not visible with this set of authorizations will fail.
- *
- * @param auths
+ */
+ public ConditionalWriterConfig setAuthorizations(Authorizations auths) {
+ ArgumentChecker.notNull(auths);
+ this.auths = auths;
+ return this;
+ }
+
+ /**
+ * Sets the maximum amount of time an unresponsive server will be re-tried. When this timeout is exceeded, the {@link ConditionalWriter} should return the
+ * mutation with an exception.<br />
+ * For no timeout, set to zero, or {@link Long#MAX_VALUE} with {@link TimeUnit#MILLISECONDS}.
+ *
+ * <p>
+ * {@link TimeUnit#MICROSECONDS} or {@link TimeUnit#NANOSECONDS} will be truncated to the nearest {@link TimeUnit#MILLISECONDS}.<br />
+ * If this truncation would result in making the value zero when it was specified as non-zero, then a minimum value of one {@link TimeUnit#MILLISECONDS} will
+ * be used.
+ *
+ * <p>
+ * <b>Default:</b> {@link Long#MAX_VALUE} (no timeout)
+ *
+ * @param timeout
+ * the timeout, in the unit specified by the value of {@code timeUnit}
+ * @param timeUnit
+ * determines how {@code timeout} will be interpreted
+ * @throws IllegalArgumentException
+ * if {@code timeout} is less than 0
+ * @return {@code this} to allow chaining of set methods
+ */
+ public ConditionalWriterConfig setTimeout(long timeout, TimeUnit timeUnit) {
+ if (timeout < 0)
+ throw new IllegalArgumentException("Negative timeout not allowed " + timeout);
+
+ if (timeout == 0)
+ this.timeout = Long.MAX_VALUE;
+ else
+ // make small, positive values that truncate to 0 when converted use the minimum millis instead
+ this.timeout = Math.max(1, timeUnit.toMillis(timeout));
+ return this;
+ }
+
+ /**
+ * Sets the maximum number of threads to use for writing data to the tablet servers.
+ *
+ * <p>
+ * <b>Default:</b> 3
+ *
+ * @param maxWriteThreads
+ * the maximum threads to use
+ * @throws IllegalArgumentException
+ * if {@code maxWriteThreads} is non-positive
+ * @return {@code this} to allow chaining of set methods
+ */
+ public ConditionalWriterConfig setMaxWriteThreads(int maxWriteThreads) {
+ if (maxWriteThreads <= 0)
+ throw new IllegalArgumentException("Max threads must be positive " + maxWriteThreads);
+
+ this.maxWriteThreads = maxWriteThreads;
+ return this;
+ }
+
+ public Authorizations getAuthorizations() {
+ return auths;
+ }
+
+ public long getTimeout(TimeUnit timeUnit) {
+ return timeUnit.convert(timeout != null ? timeout : DEFAULT_TIMEOUT, TimeUnit.MILLISECONDS);
+ }
+
+ public int getMaxWriteThreads() {
+ return maxWriteThreads != null ? maxWriteThreads : DEFAULT_MAX_WRITE_THREADS;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/Connector.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/Connector.java
index 92a1184,3189d44..4a2acff
--- a/core/src/main/java/org/apache/accumulo/core/client/Connector.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/Connector.java
@@@ -88,13 -87,12 +88,12 @@@ public abstract class Connector
* @param config
* configuration used to create batch writer
* @return BatchDeleter object for configuring and deleting
- * @throws TableNotFoundException
* @since 1.5.0
*/
-
+
public abstract BatchDeleter createBatchDeleter(String tableName, Authorizations authorizations, int numQueryThreads, BatchWriterConfig config)
throws TableNotFoundException;
-
+
/**
* Factory method to create a BatchWriter connected to Accumulo.
*
@@@ -123,12 -121,11 +122,11 @@@
* @param config
* configuration used to create batch writer
* @return BatchWriter object for configuring and writing data to
- * @throws TableNotFoundException
* @since 1.5.0
*/
-
+
public abstract BatchWriter createBatchWriter(String tableName, BatchWriterConfig config) throws TableNotFoundException;
-
+
/**
* Factory method to create a Multi-Table BatchWriter connected to Accumulo. Multi-table batch writers can queue data for multiple tables, which is good for
* ingesting data into multiple tables from the same source
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
index 7a98df2,e58a1be..e69f3dd
--- a/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
@@@ -82,11 -82,9 +82,9 @@@ public class IteratorSetting implement
public String getName() {
return name;
}
-
+
/**
* Set the iterator's name. Must be a simple alphanumeric identifier.
- *
- * @param name
*/
public void setName(String name) {
ArgumentChecker.notNull(name);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
index afa539a,29ff2a6..7b58bd4
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
@@@ -58,21 -58,17 +58,17 @@@ public interface InstanceOperations
*
* @return A map of system properties set in zookeeper. If a property is not set in zookeeper, then it will return the value set in accumulo-site.xml on some
* server. If nothing is set in an accumulo-site.xml file it will return the default value for each property.
- * @throws AccumuloException
- * @throws AccumuloSecurityException
*/
- public Map<String,String> getSystemConfiguration() throws AccumuloException, AccumuloSecurityException;
+ Map<String,String> getSystemConfiguration() throws AccumuloException, AccumuloSecurityException;
/**
*
* @return A map of system properties set in accumulo-site.xml on some server. If nothing is set in an accumulo-site.xml file it will return the default value
* for each property.
- * @throws AccumuloException
- * @throws AccumuloSecurityException
*/
- public Map<String,String> getSiteConfiguration() throws AccumuloException, AccumuloSecurityException;
+ Map<String,String> getSiteConfiguration() throws AccumuloException, AccumuloSecurityException;
/**
* List the currently active tablet servers participating in the accumulo instance
@@@ -88,11 -84,9 +84,9 @@@
* @param tserver
* The tablet server address should be of the form <ip address>:<port>
* @return A list of active scans on tablet server.
- * @throws AccumuloException
- * @throws AccumuloSecurityException
*/
- public List<ActiveScan> getActiveScans(String tserver) throws AccumuloException, AccumuloSecurityException;
+ List<ActiveScan> getActiveScans(String tserver) throws AccumuloException, AccumuloSecurityException;
/**
* List the active compaction running on a tablet server
@@@ -112,20 -104,16 +104,16 @@@
*
* @param tserver
* The tablet server address should be of the form <ip address>:<port>
- * @throws AccumuloException
* @since 1.5.0
*/
- public void ping(String tserver) throws AccumuloException;
+ void ping(String tserver) throws AccumuloException;
/**
* Test to see if the instance can load the given class as the given type. This check does not consider per table classpaths, see
* {@link TableOperations#testClassLoad(String, String, String)}
*
- * @param className
- * @param asTypeName
* @return true if the instance can load the given class as the given type, false otherwise
- * @throws AccumuloException
*/
- public boolean testClassLoad(final String className, final String asTypeName) throws AccumuloException, AccumuloSecurityException;
+ boolean testClassLoad(final String className, final String asTypeName) throws AccumuloException, AccumuloSecurityException;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
index 6166673,0823656..d9919ef
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
@@@ -109,13 -109,10 +109,10 @@@ public interface TableOperations
* Name of a table to create and import into.
* @param importDir
* Directory that contains the files copied by distcp from exportTable
- * @throws TableExistsException
- * @throws AccumuloException
- * @throws AccumuloSecurityException
* @since 1.5.0
*/
- public void importTable(String tableName, String importDir) throws TableExistsException, AccumuloException, AccumuloSecurityException;
-
+ void importTable(String tableName, String importDir) throws TableExistsException, AccumuloException, AccumuloSecurityException;
+
/**
* Exports a table. The tables data is not exported, just table metadata and a list of files to distcp. The table being exported must be offline and stay
* offline for the duration of distcp. To avoid losing access to a table it can be cloned and the clone taken offline for export.
@@@ -127,12 -124,9 +124,9 @@@
* Name of the table to export.
* @param exportDir
* An empty directory in HDFS where files containing table metadata and list of files to distcp will be placed.
- * @throws TableNotFoundException
- * @throws AccumuloException
- * @throws AccumuloSecurityException
* @since 1.5.0
*/
- public void exportTable(String tableName, String exportDir) throws TableNotFoundException, AccumuloException, AccumuloSecurityException;
+ void exportTable(String tableName, String exportDir) throws TableNotFoundException, AccumuloException, AccumuloSecurityException;
/**
* Ensures that tablets are split along a set of keys.
@@@ -212,11 -205,10 +205,10 @@@
* @throws AccumuloSecurityException
* if the user does not have permission
* @return the split points (end-row names) for the table's current split profile, grouped into fewer splits so as not to exceed maxSplits
- * @throws TableNotFoundException
* @since 1.5.0
*/
- public Collection<Text> listSplits(String tableName, int maxSplits) throws TableNotFoundException, AccumuloSecurityException, AccumuloException;
-
+ Collection<Text> listSplits(String tableName, int maxSplits) throws TableNotFoundException, AccumuloSecurityException, AccumuloException;
+
/**
* Finds the max row within a given range. To find the max row in a table, pass null for start and end row.
*
@@@ -233,14 -224,10 +224,10 @@@
* determines if the end row is included
*
* @return The max row in the range, or null if there is no visible data in the range.
- *
- * @throws AccumuloSecurityException
- * @throws AccumuloException
- * @throws TableNotFoundException
*/
- public Text getMaxRow(String tableName, Authorizations auths, Text startRow, boolean startInclusive, Text endRow, boolean endInclusive)
+ Text getMaxRow(String tableName, Authorizations auths, Text startRow, boolean startInclusive, Text endRow, boolean endInclusive)
throws TableNotFoundException, AccumuloException, AccumuloSecurityException;
-
+
/**
* Merge tablets between (start, end]
*
@@@ -401,10 -388,9 +388,9 @@@
* if a general error occurs
* @throws AccumuloSecurityException
* if the user does not have permission
- * @throws TableNotFoundException
*/
- public void flush(String tableName, Text start, Text end, boolean wait) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
-
+ void flush(String tableName, Text start, Text end, boolean wait) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
/**
* Sets a property on a table. Note that it may take a short period of time (a second) to propagate the change everywhere.
*
@@@ -527,10 -513,9 +513,9 @@@
* when there is a general accumulo error
* @throws AccumuloSecurityException
* when the user does not have the proper permissions
- * @throws TableNotFoundException
*/
- public void offline(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
-
+ void offline(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
+
/**
*
* @param tableName
@@@ -555,25 -524,9 +540,24 @@@
* when there is a general accumulo error
* @throws AccumuloSecurityException
* when the user does not have the proper permissions
- * @throws TableNotFoundException
*/
- public void online(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
-
+ void online(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
+
+ /**
+ *
+ * @param tableName
+ * the table to take online
+ * @param wait
+ * if true, then will not return until table is online
+ * @throws AccumuloException
+ * when there is a general accumulo error
+ * @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
+ * @throws TableNotFoundException
+ * @since 1.6.0
+ */
+ void online(String tableName, boolean wait) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
+
/**
* Clears the tablet locator cache for a specified table
*
@@@ -669,12 -618,9 +649,9 @@@
* @param tableName
* the name of the table
* @return a set of iterator names
- * @throws AccumuloSecurityException
- * @throws AccumuloException
- * @throws TableNotFoundException
*/
- public Map<String,EnumSet<IteratorScope>> listIterators(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
-
+ Map<String,EnumSet<IteratorScope>> listIterators(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
+
/**
* Check whether a given iterator configuration conflicts with existing configuration; in particular, determine if the name or priority are already in use for
* the specified scopes.
@@@ -700,11 -646,10 +677,10 @@@
* thrown if the constraint has already been added to the table or if there are errors in the configuration of existing constraints
* @throws AccumuloSecurityException
* thrown if the user doesn't have permission to add the constraint
- * @throws TableNotFoundException
* @since 1.5.0
*/
- public int addConstraint(String tableName, String constraintClassName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
-
+ int addConstraint(String tableName, String constraintClassName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
/**
* Remove a constraint from a table.
*
@@@ -727,23 -671,10 +702,22 @@@
* @return a map from constraint class name to assigned number
* @throws AccumuloException
* thrown if there are errors in the configuration of existing constraints
- * @throws TableNotFoundException
* @since 1.5.0
*/
- public Map<String,Integer> listConstraints(String tableName) throws AccumuloException, TableNotFoundException;
-
+ Map<String,Integer> listConstraints(String tableName) throws AccumuloException, TableNotFoundException;
+
+ /**
+ * Gets the number of bytes being used in the files for a set of tables
+ *
+ * @param tables
+ * a set of tables
+ * @return a list of disk usage objects containing linked table names and sizes
+ * @throws AccumuloException
+ * @throws AccumuloSecurityException
+ * @since 1.6.0
+ */
+ List<DiskUsage> getDiskUsage(Set<String> tables) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
/**
* Test to see if the instance can load the given class as the given type. This check uses the table classpath if it is set.
*
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
index 9d033e2,442f1be..3d69cc1
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
@@@ -1316,14 -1129,12 +1314,13 @@@ public class TableOperationsImpl extend
* when there is a general accumulo error
* @throws AccumuloSecurityException
* when the user does not have the proper permissions
- * @throws TableNotFoundException
*/
@Override
- public void offline(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
+ public void offline(String tableName, boolean wait) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
ArgumentChecker.notNull(tableName);
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)));
+ String tableId = Tables.getTableId(instance, tableName);
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)));
Map<String,String> opts = new HashMap<String,String>();
try {
@@@ -1350,13 -1153,11 +1347,12 @@@
* when there is a general accumulo error
* @throws AccumuloSecurityException
* when the user does not have the proper permissions
- * @throws TableNotFoundException
*/
@Override
- public void online(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
+ public void online(String tableName, boolean wait) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
ArgumentChecker.notNull(tableName);
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)));
+ String tableId = Tables.getTableId(instance, tableName);
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)));
Map<String,String> opts = new HashMap<String,String>();
try {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftTransportPool.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java
index ef7bcab,0000000..dad62ca
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java
@@@ -1,649 -1,0 +1,647 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapred;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.ClientSideIteratorScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableDeletedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.client.impl.OfflineScanner;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.TabletLocator;
+import org.apache.accumulo.core.client.mapreduce.InputTableConfig;
+import org.apache.accumulo.core.client.mapreduce.lib.util.InputConfigurator;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * An abstract input format to provide shared methods common to all other input format classes. At the very least, any classes inheriting from this class will
+ * need to define their own {@link RecordReader}.
+ */
+public abstract class AbstractInputFormat<K,V> implements InputFormat<K,V> {
+ protected static final Class<?> CLASS = AccumuloInputFormat.class;
+ protected static final Logger log = Logger.getLogger(CLASS);
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
+ * conversion to a string, and is not intended to be secure.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission)
+ * @param token
+ * the user's password
- * @throws org.apache.accumulo.core.client.AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public static void setConnectorInfo(JobConf job, String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ InputConfigurator.setConnectorInfo(CLASS, job, principal, token);
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * Stores the password in a file in HDFS and pulls that into the Distributed Cache in an attempt to be more secure than storing it in the Configuration.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission)
+ * @param tokenFile
+ * the path to the token file
- * @throws AccumuloSecurityException
+ * @since 1.6.0
+ */
+ public static void setConnectorInfo(JobConf job, String principal, String tokenFile) throws AccumuloSecurityException {
+ InputConfigurator.setConnectorInfo(CLASS, job, principal, tokenFile);
+ }
+
+ /**
+ * Determines if the connector has been configured.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the connector has been configured, false otherwise
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static Boolean isConnectorInfoSet(JobConf job) {
+ return InputConfigurator.isConnectorInfoSet(CLASS, job);
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static String getPrincipal(JobConf job) {
+ return InputConfigurator.getPrincipal(CLASS, job);
+ }
+
+ /**
+ * Gets the serialized token class from either the configuration or the token file.
+ *
+ * @since 1.5.0
+ * @deprecated since 1.6.0; Use {@link #getAuthenticationToken(JobConf)} instead.
+ */
+ @Deprecated
+ protected static String getTokenClass(JobConf job) {
+ return getAuthenticationToken(job).getClass().getName();
+ }
+
+ /**
+ * Gets the serialized token from either the configuration or the token file.
+ *
+ * @since 1.5.0
+ * @deprecated since 1.6.0; Use {@link #getAuthenticationToken(JobConf)} instead.
+ */
+ @Deprecated
+ protected static byte[] getToken(JobConf job) {
+ return AuthenticationToken.AuthenticationTokenSerializer.serialize(getAuthenticationToken(job));
+ }
+
+ /**
+ * Gets the authenticated token from either the specified token file or directly from the configuration, whichever was used when the job was configured.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the principal's authentication token
+ * @since 1.6.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ * @see #setConnectorInfo(JobConf, String, String)
+ */
+ protected static AuthenticationToken getAuthenticationToken(JobConf job) {
+ return InputConfigurator.getAuthenticationToken(CLASS, job);
+ }
+
+ /**
+ * Configures a {@link org.apache.accumulo.core.client.ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @param zooKeepers
+ * a comma-separated list of zookeeper servers
+ * @since 1.5.0
+ * @deprecated since 1.6.0; Use {@link #setZooKeeperInstance(JobConf, ClientConfiguration)} instead.
+ */
+ @Deprecated
+ public static void setZooKeeperInstance(JobConf job, String instanceName, String zooKeepers) {
+ InputConfigurator.setZooKeeperInstance(CLASS, job, instanceName, zooKeepers);
+ }
+
+ /**
+ * Configures a {@link org.apache.accumulo.core.client.ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param clientConfig
+ * client configuration containing connection options
+ * @since 1.6.0
+ */
+ public static void setZooKeeperInstance(JobConf job, ClientConfiguration clientConfig) {
+ InputConfigurator.setZooKeeperInstance(CLASS, job, clientConfig);
+ }
+
+ /**
+ * Configures a {@link org.apache.accumulo.core.client.mock.MockInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @since 1.5.0
+ */
+ public static void setMockInstance(JobConf job, String instanceName) {
+ InputConfigurator.setMockInstance(CLASS, job, instanceName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link org.apache.accumulo.core.client.Instance} based on the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return an Accumulo instance
+ * @since 1.5.0
+ * @see #setZooKeeperInstance(JobConf, String, String)
+ * @see #setMockInstance(JobConf, String)
+ */
+ protected static Instance getInstance(JobConf job) {
+ return InputConfigurator.getInstance(CLASS, job);
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param level
+ * the logging level
+ * @since 1.5.0
+ */
+ public static void setLogLevel(JobConf job, Level level) {
+ InputConfigurator.setLogLevel(CLASS, job, level);
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the log level
+ * @since 1.5.0
+ * @see #setLogLevel(JobConf, Level)
+ */
+ protected static Level getLogLevel(JobConf job) {
+ return InputConfigurator.getLogLevel(CLASS, job);
+ }
+
+ /**
+ * Sets the {@link org.apache.accumulo.core.security.Authorizations} used to scan. Must be a subset of the user's authorization. Defaults to the empty set.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param auths
+ * the user's authorizations
+ * @since 1.5.0
+ */
+ public static void setScanAuthorizations(JobConf job, Authorizations auths) {
+ InputConfigurator.setScanAuthorizations(CLASS, job, auths);
+ }
+
+ /**
+ * Gets the authorizations to set for the scans from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the Accumulo scan authorizations
+ * @since 1.5.0
+ * @see #setScanAuthorizations(JobConf, Authorizations)
+ */
+ protected static Authorizations getScanAuthorizations(JobConf job) {
+ return InputConfigurator.getScanAuthorizations(CLASS, job);
+ }
+
+ /**
+ * Initializes an Accumulo {@link org.apache.accumulo.core.client.impl.TabletLocator} based on the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return an Accumulo tablet locator
+ * @throws org.apache.accumulo.core.client.TableNotFoundException
+ * if the table name set on the configuration doesn't exist
+ * @since 1.6.0
+ */
+ protected static TabletLocator getTabletLocator(JobConf job, String tableId) throws TableNotFoundException {
+ return InputConfigurator.getTabletLocator(CLASS, job, tableId);
+ }
+
+ // InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext job)
+ /**
+ * Check whether a configuration is fully configured to be used with an Accumulo {@link InputFormat}.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @throws java.io.IOException
+ * if the context is improperly configured
+ * @since 1.5.0
+ */
+ protected static void validateOptions(JobConf job) throws IOException {
+ InputConfigurator.validateOptions(CLASS, job);
+ }
+
+ /**
+ * Fetches all {@link InputTableConfig}s that have been set on the given Hadoop job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @return the {@link InputTableConfig} objects set on the job
+ * @since 1.6.0
+ */
+ public static Map<String,InputTableConfig> getInputTableConfigs(JobConf job) {
+ return InputConfigurator.getInputTableConfigs(CLASS, job);
+ }
+
+ /**
+ * Fetches a {@link InputTableConfig} that has been set on the configuration for a specific table.
+ *
+ * <p>
+ * null is returned in the event that the table doesn't exist.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param tableName
+ * the table name for which to grab the config object
+ * @return the {@link InputTableConfig} for the given table
+ * @since 1.6.0
+ */
+ public static InputTableConfig getInputTableConfig(JobConf job, String tableName) {
+ return InputConfigurator.getInputTableConfig(CLASS, job, tableName);
+ }
+
+ /**
+ * An abstract base class to be used to create {@link org.apache.hadoop.mapred.RecordReader} instances that convert from Accumulo
+ * {@link org.apache.accumulo.core.data.Key}/{@link org.apache.accumulo.core.data.Value} pairs to the user's K/V types.
+ *
+ * Subclasses must implement {@link #next(Object, Object)} to update key and value, and also to update the following variables:
+ * <ul>
+ * <li>Key {@link #currentKey} (used for progress reporting)</li>
+ * <li>int {@link #numKeysRead} (used for progress reporting)</li>
+ * </ul>
+ */
+ protected abstract static class AbstractRecordReader<K,V> implements RecordReader<K,V> {
+ protected long numKeysRead;
+ protected Iterator<Map.Entry<Key,Value>> scannerIterator;
+ protected RangeInputSplit split;
+
+ /**
+ * Configures the iterators on a scanner for the given table name.
+ *
+ * @param job
+ * the Hadoop job configuration
+ * @param scanner
+ * the scanner for which to configure the iterators
+ * @param tableName
+ * the table name for which the scanner is configured
+ * @since 1.6.0
+ */
+ protected abstract void setupIterators(JobConf job, Scanner scanner, String tableName, RangeInputSplit split);
+
+ /**
+ * Initialize a scanner over the given input split using this task attempt configuration.
+ */
+ public void initialize(InputSplit inSplit, JobConf job) throws IOException {
+ Scanner scanner;
+ split = (RangeInputSplit) inSplit;
+ log.debug("Initializing input split: " + split.getRange());
+
+ Instance instance = split.getInstance();
+ if (null == instance) {
+ instance = getInstance(job);
+ }
+
+ String principal = split.getPrincipal();
+ if (null == principal) {
+ principal = getPrincipal(job);
+ }
+
+ AuthenticationToken token = split.getToken();
+ if (null == token) {
+ token = getAuthenticationToken(job);
+ }
+
+ Authorizations authorizations = split.getAuths();
+ if (null == authorizations) {
+ authorizations = getScanAuthorizations(job);
+ }
+
+ String table = split.getTableName();
+
+ // in case the table name changed, we can still use the previous name for terms of configuration,
+ // but the scanner will use the table id resolved at job setup time
+ InputTableConfig tableConfig = getInputTableConfig(job, split.getTableName());
+
+ Boolean isOffline = split.isOffline();
+ if (null == isOffline) {
+ isOffline = tableConfig.isOfflineScan();
+ }
+
+ Boolean isIsolated = split.isIsolatedScan();
+ if (null == isIsolated) {
+ isIsolated = tableConfig.shouldUseIsolatedScanners();
+ }
+
+ Boolean usesLocalIterators = split.usesLocalIterators();
+ if (null == usesLocalIterators) {
+ usesLocalIterators = tableConfig.shouldUseLocalIterators();
+ }
+
+ List<IteratorSetting> iterators = split.getIterators();
+ if (null == iterators) {
+ iterators = tableConfig.getIterators();
+ }
+
+ Collection<Pair<Text,Text>> columns = split.getFetchedColumns();
+ if (null == columns) {
+ columns = tableConfig.getFetchedColumns();
+ }
+
+ try {
+ log.debug("Creating connector with user: " + principal);
+ log.debug("Creating scanner for table: " + table);
+ log.debug("Authorizations are: " + authorizations);
+ if (isOffline) {
+ scanner = new OfflineScanner(instance, new Credentials(principal, token), split.getTableId(), authorizations);
+ } else if (instance instanceof MockInstance) {
+ scanner = instance.getConnector(principal, token).createScanner(split.getTableName(), authorizations);
+ } else {
+ scanner = new ScannerImpl(instance, new Credentials(principal, token), split.getTableId(), authorizations);
+ }
+ if (isIsolated) {
+ log.info("Creating isolated scanner");
+ scanner = new IsolatedScanner(scanner);
+ }
+ if (usesLocalIterators) {
+ log.info("Using local iterators");
+ scanner = new ClientSideIteratorScanner(scanner);
+ }
+ setupIterators(job, scanner, split.getTableName(), split);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+
+ // setup a scanner within the bounds of this split
+ for (Pair<Text,Text> c : columns) {
+ if (c.getSecond() != null) {
+ log.debug("Fetching column " + c.getFirst() + ":" + c.getSecond());
+ scanner.fetchColumn(c.getFirst(), c.getSecond());
+ } else {
+ log.debug("Fetching column family " + c.getFirst());
+ scanner.fetchColumnFamily(c.getFirst());
+ }
+ }
+
+ scanner.setRange(split.getRange());
+
+ numKeysRead = 0;
+
+ // do this last after setting all scanner options
+ scannerIterator = scanner.iterator();
+ }
+
+ @Override
+ public void close() {}
+
+ @Override
+ public long getPos() throws IOException {
+ return numKeysRead;
+ }
+
+ @Override
+ public float getProgress() throws IOException {
+ if (numKeysRead > 0 && currentKey == null)
+ return 1.0f;
+ return split.getProgress(currentKey);
+ }
+
+ protected Key currentKey = null;
+
+ }
+
+ Map<String,Map<KeyExtent,List<Range>>> binOfflineTable(JobConf job, String tableId, List<Range> ranges) throws TableNotFoundException, AccumuloException,
+ AccumuloSecurityException {
+
+ Instance instance = getInstance(job);
+ Connector conn = instance.getConnector(getPrincipal(job), getAuthenticationToken(job));
+
+ return InputConfigurator.binOffline(tableId, ranges, instance, conn);
+ }
+
+ /**
+ * Read the metadata table to get tablets and match up ranges to them.
+ */
+ @Override
+ public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
+ Level logLevel = getLogLevel(job);
+ log.setLevel(logLevel);
+ validateOptions(job);
+
+ Random random = new Random();
+ LinkedList<InputSplit> splits = new LinkedList<InputSplit>();
+ Map<String,InputTableConfig> tableConfigs = getInputTableConfigs(job);
+ for (Map.Entry<String,InputTableConfig> tableConfigEntry : tableConfigs.entrySet()) {
+ String tableName = tableConfigEntry.getKey();
+ InputTableConfig tableConfig = tableConfigEntry.getValue();
+
+ Instance instance = getInstance(job);
+ boolean mockInstance;
+ String tableId;
+ // resolve table name to id once, and use id from this point forward
+ if (instance instanceof MockInstance) {
+ tableId = "";
+ mockInstance = true;
+ } else {
+ try {
+ tableId = Tables.getTableId(instance, tableName);
+ } catch (TableNotFoundException e) {
+ throw new IOException(e);
+ }
+ mockInstance = false;
+ }
+
+ Authorizations auths = getScanAuthorizations(job);
+ String principal = getPrincipal(job);
+ AuthenticationToken token = getAuthenticationToken(job);
+
+ boolean autoAdjust = tableConfig.shouldAutoAdjustRanges();
+ List<Range> ranges = autoAdjust ? Range.mergeOverlapping(tableConfig.getRanges()) : tableConfig.getRanges();
+ if (ranges.isEmpty()) {
+ ranges = new ArrayList<Range>(1);
+ ranges.add(new Range());
+ }
+
+ // get the metadata information for these ranges
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+ TabletLocator tl;
+ try {
+ if (tableConfig.isOfflineScan()) {
+ binnedRanges = binOfflineTable(job, tableId, ranges);
+ while (binnedRanges == null) {
+ // Some tablets were still online, try again
+ UtilWaitThread.sleep(100 + random.nextInt(100)); // sleep randomly between 100 and 200 ms
+ binnedRanges = binOfflineTable(job, tableId, ranges);
+ }
+ } else {
+ tl = getTabletLocator(job, tableId);
+ // its possible that the cache could contain complete, but old information about a tables tablets... so clear it
+ tl.invalidateCache();
+ Credentials creds = new Credentials(getPrincipal(job), getAuthenticationToken(job));
+
+ while (!tl.binRanges(creds, ranges, binnedRanges).isEmpty()) {
+ if (!(instance instanceof MockInstance)) {
+ if (!Tables.exists(instance, tableId))
+ throw new TableDeletedException(tableId);
+ if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
+ throw new TableOfflineException(instance, tableId);
+ }
+ binnedRanges.clear();
+ log.warn("Unable to locate bins for specified ranges. Retrying.");
+ UtilWaitThread.sleep(100 + random.nextInt(100)); // sleep randomly between 100 and 200 ms
+ tl.invalidateCache();
+ }
+ }
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ HashMap<Range,ArrayList<String>> splitsToAdd = null;
+
+ if (!autoAdjust)
+ splitsToAdd = new HashMap<Range,ArrayList<String>>();
+
+ HashMap<String,String> hostNameCache = new HashMap<String,String>();
+ for (Map.Entry<String,Map<KeyExtent,List<Range>>> tserverBin : binnedRanges.entrySet()) {
+ String ip = tserverBin.getKey().split(":", 2)[0];
+ String location = hostNameCache.get(ip);
+ if (location == null) {
+ InetAddress inetAddress = InetAddress.getByName(ip);
+ location = inetAddress.getCanonicalHostName();
+ hostNameCache.put(ip, location);
+ }
+ for (Map.Entry<KeyExtent,List<Range>> extentRanges : tserverBin.getValue().entrySet()) {
+ Range ke = extentRanges.getKey().toDataRange();
+ for (Range r : extentRanges.getValue()) {
+ if (autoAdjust) {
+ // divide ranges into smaller ranges, based on the tablets
+ RangeInputSplit split = new RangeInputSplit(tableName, tableId, ke.clip(r), new String[] {location});
+
+ split.setOffline(tableConfig.isOfflineScan());
+ split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
+ split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());
+ split.setMockInstance(mockInstance);
+ split.setFetchedColumns(tableConfig.getFetchedColumns());
+ split.setPrincipal(principal);
+ split.setToken(token);
+ split.setInstanceName(instance.getInstanceName());
+ split.setZooKeepers(instance.getZooKeepers());
+ split.setAuths(auths);
+ split.setIterators(tableConfig.getIterators());
+ split.setLogLevel(logLevel);
+
+ splits.add(split);
+ } else {
+ // don't divide ranges
+ ArrayList<String> locations = splitsToAdd.get(r);
+ if (locations == null)
+ locations = new ArrayList<String>(1);
+ locations.add(location);
+ splitsToAdd.put(r, locations);
+ }
+ }
+ }
+ }
+
+ if (!autoAdjust)
+ for (Map.Entry<Range,ArrayList<String>> entry : splitsToAdd.entrySet()) {
+ RangeInputSplit split = new RangeInputSplit(tableName, tableId, entry.getKey(), entry.getValue().toArray(new String[0]));
+
+ split.setOffline(tableConfig.isOfflineScan());
+ split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
+ split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());
+ split.setMockInstance(mockInstance);
+ split.setFetchedColumns(tableConfig.getFetchedColumns());
+ split.setPrincipal(principal);
+ split.setToken(token);
+ split.setInstanceName(instance.getInstanceName());
+ split.setZooKeepers(instance.getZooKeepers());
+ split.setAuths(auths);
+ split.setIterators(tableConfig.getIterators());
+ split.setLogLevel(logLevel);
+
+ splits.add(split);
+ }
+ }
+
+ return splits.toArray(new InputSplit[splits.size()]);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
index 02512a4,d7be37c..1ec4c41
--- a/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
@@@ -91,26 -88,7 +90,25 @@@ public class AccumuloOutputFormat imple
public static void setConnectorInfo(JobConf job, String principal, AuthenticationToken token) throws AccumuloSecurityException {
OutputConfigurator.setConnectorInfo(CLASS, job, principal, token);
}
-
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * Stores the password in a file in HDFS and pulls that into the Distributed Cache in an attempt to be more secure than storing it in the Configuration.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission if {@link #setCreateTables(JobConf, boolean)} is set to true)
+ * @param tokenFile
+ * the path to the password file
- * @throws AccumuloSecurityException
+ * @since 1.6.0
+ */
+ public static void setConnectorInfo(JobConf job, String principal, String tokenFile) throws AccumuloSecurityException {
+ OutputConfigurator.setConnectorInfo(CLASS, job, principal, tokenFile);
+ }
+
/**
* Determines if the connector has been configured.
*
[06/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --cc minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
index a366c16,0000000..e2b2f83
mode 100644,000000..100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
@@@ -1,392 -1,0 +1,382 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.minicluster;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.gc.SimpleGarbageCollector;
+import org.apache.accumulo.server.master.Master;
+import org.apache.accumulo.server.tabletserver.TabletServer;
+import org.apache.accumulo.server.util.Initialize;
+import org.apache.accumulo.server.util.PortUtils;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.start.Main;
+import org.apache.zookeeper.server.ZooKeeperServerMain;
+
+/**
+ * A utility class that will create Zookeeper and Accumulo processes that write all of their data to a single local directory. This class makes it easy to test
+ * code against a real Accumulo instance. Its much more accurate for testing than MockAccumulo, but much slower than MockAccumulo.
+ *
+ * @since 1.5.0
+ */
+public class MiniAccumuloCluster {
+
+ private static final String INSTANCE_SECRET = "DONTTELL";
+ private static final String INSTANCE_NAME = "miniInstance";
+
+ private static class LogWriter extends Thread {
+ private BufferedReader in;
+ private BufferedWriter out;
+
- /**
- * @throws IOException
- */
+ public LogWriter(InputStream stream, File logFile) throws IOException {
+ this.setDaemon(true);
+ this.in = new BufferedReader(new InputStreamReader(stream, Constants.UTF8));
+ out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(logFile), Constants.UTF8));
+
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ flush();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }, 1000, 1000);
+ }
+
+ public synchronized void flush() throws IOException {
+ if (out != null)
+ out.flush();
+ }
+
+ @Override
+ public void run() {
+ String line;
+
+ try {
+ while ((line = in.readLine()) != null) {
+ out.append(line);
+ out.append("\n");
+ }
+
+ synchronized (this) {
+ out.close();
+ out = null;
+ in.close();
+ }
+
+ } catch (IOException e) {}
+ }
+ }
+
+ private File libDir;
+ private File libExtDir;
+ private File confDir;
+ private File zooKeeperDir;
+ private File accumuloDir;
+ private File zooCfgFile;
+ private File logDir;
+ private File walogDir;
+
+ private Process zooKeeperProcess;
+ private Process masterProcess;
+ private Process gcProcess;
+
+ private int zooKeeperPort;
+
+ private List<LogWriter> logWriters = new ArrayList<MiniAccumuloCluster.LogWriter>();
+
+ private MiniAccumuloConfig config;
+ private Process[] tabletServerProcesses;
+
+ private Process exec(Class<? extends Object> clazz, String... args) throws IOException {
+ String javaHome = System.getProperty("java.home");
+ String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
+ String classpath = System.getProperty("java.class.path");
+
+ classpath = confDir.getAbsolutePath() + File.pathSeparator + classpath;
+
+ String className = clazz.getCanonicalName();
+
+ ArrayList<String> argList = new ArrayList<String>();
+
+ argList.addAll(Arrays.asList(javaBin, "-cp", classpath, "-Xmx128m", "-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75",
+ "-Dapple.awt.UIElement=true", Main.class.getName(), className));
+
+ argList.addAll(Arrays.asList(args));
+
+ ProcessBuilder builder = new ProcessBuilder(argList);
+
+ builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
+ builder.environment().put("ACCUMULO_LOG_DIR", logDir.getAbsolutePath());
+
+ // if we're running under accumulo.start, we forward these env vars
+ String env = System.getenv("HADOOP_PREFIX");
+ if (env != null)
+ builder.environment().put("HADOOP_PREFIX", env);
+ env = System.getenv("ZOOKEEPER_HOME");
+ if (env != null)
+ builder.environment().put("ZOOKEEPER_HOME", env);
+
+ Process process = builder.start();
+
+ LogWriter lw;
+ lw = new LogWriter(process.getErrorStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".err"));
+ logWriters.add(lw);
+ lw.start();
+ lw = new LogWriter(process.getInputStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".out"));
+ logWriters.add(lw);
+ lw.start();
+
+ return process;
+ }
+
+ private void appendProp(Writer fileWriter, Property key, String value, Map<String,String> siteConfig) throws IOException {
+ appendProp(fileWriter, key.getKey(), value, siteConfig);
+ }
+
+ private void appendProp(Writer fileWriter, String key, String value, Map<String,String> siteConfig) throws IOException {
+ if (!siteConfig.containsKey(key))
+ fileWriter.append("<property><name>" + key + "</name><value>" + value + "</value></property>\n");
+ }
+
+ /**
+ * Sets a given key with a random port for the value on the site config if it doesn't already exist.
+ */
+ private void mergePropWithRandomPort(Map<String,String> siteConfig, String key) {
+ if (!siteConfig.containsKey(key)) {
+ siteConfig.put(key, "0");
+ }
+ }
+
+ /**
+ *
+ * @param dir
+ * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
+ * and Junit provide methods for creating temporary directories.
+ * @param rootPassword
+ * Initial root password for instance.
- * @throws IOException
+ */
+ public MiniAccumuloCluster(File dir, String rootPassword) throws IOException {
+ this(new MiniAccumuloConfig(dir, rootPassword));
+ }
+
+ /**
+ * @param config
+ * initial configuration
- * @throws IOException
+ */
+
+ public MiniAccumuloCluster(MiniAccumuloConfig config) throws IOException {
+
+ if (config.getDir().exists() && !config.getDir().isDirectory())
+ throw new IllegalArgumentException("Must pass in directory, " + config.getDir() + " is a file");
+
+ if (config.getDir().exists() && config.getDir().list().length != 0)
+ throw new IllegalArgumentException("Directory " + config.getDir() + " is not empty");
+
+ this.config = config;
+
+ libDir = new File(config.getDir(), "lib");
+ libExtDir = new File(libDir, "ext");
+ confDir = new File(config.getDir(), "conf");
+ accumuloDir = new File(config.getDir(), "accumulo");
+ zooKeeperDir = new File(config.getDir(), "zookeeper");
+ logDir = new File(config.getDir(), "logs");
+ walogDir = new File(config.getDir(), "walogs");
+
+ confDir.mkdirs();
+ accumuloDir.mkdirs();
+ zooKeeperDir.mkdirs();
+ logDir.mkdirs();
+ walogDir.mkdirs();
+ libDir.mkdirs();
+
+ // Avoid the classloader yelling that the general.dynamic.classpaths value is invalid because
+ // $ACCUMULO_HOME/lib/ext isn't defined.
+ libExtDir.mkdirs();
+
+ zooKeeperPort = PortUtils.getRandomFreePort();
+
+ File siteFile = new File(confDir, "accumulo-site.xml");
+
+ OutputStreamWriter fileWriter = new OutputStreamWriter(new FileOutputStream(siteFile), Constants.UTF8);
+ fileWriter.append("<configuration>\n");
+
+ HashMap<String,String> siteConfig = new HashMap<String,String>(config.getSiteConfig());
+
+ appendProp(fileWriter, Property.INSTANCE_DFS_URI, "file:///", siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_DFS_DIR, accumuloDir.getAbsolutePath(), siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_ZK_HOST, "localhost:" + zooKeeperPort, siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_SECRET, INSTANCE_SECRET, siteConfig);
+ appendProp(fileWriter, Property.TSERV_PORTSEARCH, "true", siteConfig);
+ appendProp(fileWriter, Property.LOGGER_DIR, walogDir.getAbsolutePath(), siteConfig);
+ appendProp(fileWriter, Property.TSERV_DATACACHE_SIZE, "10M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_INDEXCACHE_SIZE, "10M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_MAXMEM, "50M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_WALOG_MAX_SIZE, "100M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_NATIVEMAP_ENABLED, "false", siteConfig);
+ appendProp(fileWriter, Property.TRACE_TOKEN_PROPERTY_PREFIX + ".password", config.getRootPassword(), siteConfig);
+ appendProp(fileWriter, Property.GC_CYCLE_DELAY, "4s", siteConfig);
+ appendProp(fileWriter, Property.GC_CYCLE_START, "0s", siteConfig);
+ mergePropWithRandomPort(siteConfig, Property.MASTER_CLIENTPORT.getKey());
+ mergePropWithRandomPort(siteConfig, Property.TRACE_PORT.getKey());
+ mergePropWithRandomPort(siteConfig, Property.TSERV_CLIENTPORT.getKey());
+ mergePropWithRandomPort(siteConfig, Property.MONITOR_PORT.getKey());
+ mergePropWithRandomPort(siteConfig, Property.GC_PORT.getKey());
+
+ // since there is a small amount of memory, check more frequently for majc... setting may not be needed in 1.5
+ appendProp(fileWriter, Property.TSERV_MAJC_DELAY, "3", siteConfig);
+
+ // ACCUMULO-1472 -- Use the classpath, not what might be installed on the system.
+ // We have to set *something* here, otherwise the AccumuloClassLoader will default to pulling from
+ // environment variables (e.g. ACCUMULO_HOME, HADOOP_HOME/PREFIX) which will result in multiple copies
+ // of artifacts on the classpath as they'll be provided by the invoking application
+ appendProp(fileWriter, Property.GENERAL_CLASSPATHS, libDir.getAbsolutePath() + "/[^.].*.jar", siteConfig);
+ appendProp(fileWriter, Property.GENERAL_DYNAMIC_CLASSPATHS, libExtDir.getAbsolutePath() + "/[^.].*.jar", siteConfig);
+
+ for (Entry<String,String> entry : siteConfig.entrySet())
+ fileWriter.append("<property><name>" + entry.getKey() + "</name><value>" + entry.getValue() + "</value></property>\n");
+ fileWriter.append("</configuration>\n");
+ fileWriter.close();
+
+ zooCfgFile = new File(confDir, "zoo.cfg");
+ fileWriter = new OutputStreamWriter(new FileOutputStream(zooCfgFile), Constants.UTF8);
+
+ // zookeeper uses Properties to read its config, so use that to write in order to properly escape things like Windows paths
+ Properties zooCfg = new Properties();
+ zooCfg.setProperty("tickTime", "1000");
+ zooCfg.setProperty("initLimit", "10");
+ zooCfg.setProperty("syncLimit", "5");
+ zooCfg.setProperty("clientPort", zooKeeperPort + "");
+ zooCfg.setProperty("maxClientCnxns", "100");
+ zooCfg.setProperty("dataDir", zooKeeperDir.getAbsolutePath());
+ zooCfg.store(fileWriter, null);
+
+ fileWriter.close();
+
+ }
+
+ /**
+ * Starts Accumulo and Zookeeper processes. Can only be called once.
+ *
- * @throws IOException
- * @throws InterruptedException
+ * @throws IllegalStateException
+ * if already started
+ */
+
+ public void start() throws IOException, InterruptedException {
+ if (zooKeeperProcess != null)
+ throw new IllegalStateException("Already started");
+
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ try {
+ MiniAccumuloCluster.this.stop();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+
+ zooKeeperProcess = exec(Main.class, ZooKeeperServerMain.class.getName(), zooCfgFile.getAbsolutePath());
+
+ // sleep a little bit to let zookeeper come up before calling init, seems to work better
+ UtilWaitThread.sleep(250);
+
+ Process initProcess = exec(Initialize.class, "--instance-name", INSTANCE_NAME, "--password", config.getRootPassword());
+ int ret = initProcess.waitFor();
+ if (ret != 0) {
+ throw new RuntimeException("Initialize process returned " + ret + ". Check the logs in " + logDir + " for errors.");
+ }
+
+ tabletServerProcesses = new Process[config.getNumTservers()];
+ for (int i = 0; i < config.getNumTservers(); i++) {
+ tabletServerProcesses[i] = exec(TabletServer.class);
+ }
+
+ masterProcess = exec(Master.class);
+
+ gcProcess = exec(SimpleGarbageCollector.class);
+ }
+
+ /**
+ * @return Accumulo instance name
+ */
+
+ public String getInstanceName() {
+ return INSTANCE_NAME;
+ }
+
+ /**
+ * @return zookeeper connection string
+ */
+
+ public String getZooKeepers() {
+ return "localhost:" + zooKeeperPort;
+ }
+
+ /**
+ * Stops Accumulo and Zookeeper processes. If stop is not called, there is a shutdown hook that is setup to kill the processes. Howerver its probably best to
+ * call stop in a finally block as soon as possible.
- *
- * @throws IOException
- * @throws InterruptedException
+ */
+
+ public void stop() throws IOException, InterruptedException {
+ if (zooKeeperProcess != null) {
+ zooKeeperProcess.destroy();
+ zooKeeperProcess.waitFor();
+ }
+ if (masterProcess != null) {
+ masterProcess.destroy();
+ masterProcess.waitFor();
+ }
+ if (tabletServerProcesses != null) {
+ for (Process tserver : tabletServerProcesses) {
+ tserver.destroy();
+ tserver.waitFor();
+ }
+ }
+
+ for (LogWriter lw : logWriters)
+ lw.flush();
+
+ if (gcProcess != null) {
+ gcProcess.destroy();
+ gcProcess.waitFor();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/proxy/src/test/java/org/apache/accumulo/proxy/TestProxyReadWrite.java
----------------------------------------------------------------------
diff --cc proxy/src/test/java/org/apache/accumulo/proxy/TestProxyReadWrite.java
index 99a3218,0000000..c0049a0
mode 100644,000000..100644
--- a/proxy/src/test/java/org/apache/accumulo/proxy/TestProxyReadWrite.java
+++ b/proxy/src/test/java/org/apache/accumulo/proxy/TestProxyReadWrite.java
@@@ -1,488 -1,0 +1,478 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.proxy;
+
+import static org.junit.Assert.assertEquals;
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.proxy.thrift.BatchScanOptions;
+import org.apache.accumulo.proxy.thrift.ColumnUpdate;
+import org.apache.accumulo.proxy.thrift.IteratorSetting;
+import org.apache.accumulo.proxy.thrift.Key;
+import org.apache.accumulo.proxy.thrift.KeyValue;
+import org.apache.accumulo.proxy.thrift.Range;
+import org.apache.accumulo.proxy.thrift.ScanColumn;
+import org.apache.accumulo.proxy.thrift.ScanOptions;
+import org.apache.accumulo.proxy.thrift.ScanResult;
+import org.apache.accumulo.proxy.thrift.TimeType;
+import org.apache.thrift.protocol.TCompactProtocol;
+import org.apache.thrift.server.TServer;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestProxyReadWrite {
+ protected static TServer proxy;
+ protected static Thread thread;
+ protected static TestProxyClient tpc;
+ protected static ByteBuffer userpass;
+ protected static final int port = 10194;
+ protected static final String testtable = "testtable";
+
+ @SuppressWarnings("serial")
+ @BeforeClass
+ public static void setup() throws Exception {
+ Properties prop = new Properties();
+ prop.setProperty("useMockInstance", "true");
+ prop.put("tokenClass", PasswordToken.class.getName());
+
+ proxy = Proxy.createProxyServer(Class.forName("org.apache.accumulo.proxy.thrift.AccumuloProxy"), Class.forName("org.apache.accumulo.proxy.ProxyServer"),
+ port, TCompactProtocol.Factory.class, prop);
+ thread = new Thread() {
+ @Override
+ public void run() {
+ proxy.serve();
+ }
+ };
+ thread.start();
+ tpc = new TestProxyClient("localhost", port);
+ userpass = tpc.proxy().login("root", new TreeMap<String, String>() {{put("password",""); }});
+ }
+
+ @AfterClass
+ public static void tearDown() throws InterruptedException {
+ proxy.stop();
+ thread.join();
+ }
+
+ @Before
+ public void makeTestTable() throws Exception {
+ tpc.proxy().createTable(userpass, testtable, true, TimeType.MILLIS);
+ }
+
+ @After
+ public void deleteTestTable() throws Exception {
+ tpc.proxy().deleteTable(userpass, testtable);
+ }
+
+ private static void addMutation(Map<ByteBuffer,List<ColumnUpdate>> mutations, String row, String cf, String cq, String value) {
+ ColumnUpdate update = new ColumnUpdate(ByteBuffer.wrap(cf.getBytes()), ByteBuffer.wrap(cq.getBytes()));
+ update.setValue(value.getBytes());
+ mutations.put(ByteBuffer.wrap(row.getBytes()), Collections.singletonList(update));
+ }
+
+ private static void addMutation(Map<ByteBuffer,List<ColumnUpdate>> mutations, String row, String cf, String cq, String vis, String value) {
+ ColumnUpdate update = new ColumnUpdate(ByteBuffer.wrap(cf.getBytes()), ByteBuffer.wrap(cq.getBytes()));
+ update.setValue(value.getBytes());
+ update.setColVisibility(vis.getBytes());
+ mutations.put(ByteBuffer.wrap(row.getBytes()), Collections.singletonList(update));
+ }
+
+ /**
+ * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Set a range so only the entries between -Inf...5 come back (there should be
+ * 50,000)
- *
- * @throws Exception
+ */
+ @Test
+ public void readWriteBatchOneShotWithRange() throws Exception {
+ int maxInserts = 100000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ for (int i = 0; i < maxInserts; i++) {
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+ tpc.proxy().updateAndFlush(userpass, testtable, mutations);
+ mutations.clear();
+ }
+ }
+
+ Key stop = new Key();
+ stop.setRow("5".getBytes());
+ BatchScanOptions options = new BatchScanOptions();
+ options.ranges = Collections.singletonList(new Range(null, false, stop, false));
+ String cookie = tpc.proxy().createBatchScanner(userpass, testtable, options);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ i += kvList.getResultsSize();
+ hasNext = kvList.isMore();
+ }
+ assertEquals(i, 50000);
+ }
+
+ /**
+ * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Set a columnFamily so only the entries with specified column family come back (there should be
+ * 50,000)
- *
- * @throws Exception
+ */
+ @Test
+ public void readWriteBatchOneShotWithColumnFamilyOnly() throws Exception {
+ int maxInserts = 100000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ for (int i = 0; i < maxInserts; i++) {
+
+ addMutation(mutations, String.format(format, i), "cf" + (i % 2) , "cq" + (i % 2), Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+ tpc.proxy().updateAndFlush(userpass, testtable, mutations);
+ mutations.clear();
+ }
+ }
+
+ BatchScanOptions options = new BatchScanOptions();
+
+ ScanColumn sc = new ScanColumn();
+ sc.colFamily = ByteBuffer.wrap("cf0".getBytes());
+
+ options.columns = Collections.singletonList(sc);
+ String cookie = tpc.proxy().createBatchScanner(userpass, testtable, options);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ i += kvList.getResultsSize();
+ hasNext = kvList.isMore();
+ }
+ assertEquals(i, 50000);
+ }
+
+
+ /**
+ * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Set a columnFamily + columnQualififer so only the entries with specified column
+ * come back (there should be 50,000)
- *
- * @throws Exception
+ */
+ @Test
+ public void readWriteBatchOneShotWithFullColumn() throws Exception {
+ int maxInserts = 100000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ for (int i = 0; i < maxInserts; i++) {
+
+ addMutation(mutations, String.format(format, i), "cf" + (i % 2) , "cq" + (i % 2), Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+ tpc.proxy().updateAndFlush(userpass, testtable, mutations);
+ mutations.clear();
+ }
+ }
+
+ BatchScanOptions options = new BatchScanOptions();
+
+ ScanColumn sc = new ScanColumn();
+ sc.colFamily = ByteBuffer.wrap("cf0".getBytes());
+ sc.colQualifier = ByteBuffer.wrap("cq0".getBytes());
+
+ options.columns = Collections.singletonList(sc);
+ String cookie = tpc.proxy().createBatchScanner(userpass, testtable, options);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ i += kvList.getResultsSize();
+ hasNext = kvList.isMore();
+ }
+ assertEquals(i, 50000);
+ }
+
+
+ /**
+ * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Filter the results so only the even numbers come back.
- *
- * @throws Exception
+ */
+ @Test
+ public void readWriteBatchOneShotWithFilterIterator() throws Exception {
+ int maxInserts = 10000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ for (int i = 0; i < maxInserts; i++) {
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+ tpc.proxy().updateAndFlush(userpass, testtable, mutations);
+ mutations.clear();
+ }
+
+ }
+
+ String regex = ".*[02468]";
+
+ org.apache.accumulo.core.client.IteratorSetting is = new org.apache.accumulo.core.client.IteratorSetting(50, regex, RegExFilter.class);
+ RegExFilter.setRegexs(is, regex, null, null, null, false);
+
+ IteratorSetting pis = Util.iteratorSetting2ProxyIteratorSetting(is);
+ ScanOptions opts = new ScanOptions();
+ opts.iterators = Collections.singletonList(pis);
+ String cookie = tpc.proxy().createScanner(userpass, testtable, opts);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ for (KeyValue kv : kvList.getResults()) {
+ assertEquals(Integer.parseInt(new String(kv.getKey().getRow())), i);
+
+ i += 2;
+ }
+ hasNext = kvList.isMore();
+ }
+ }
+
+ @Test
+ public void readWriteOneShotWithRange() throws Exception {
+ int maxInserts = 100000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ for (int i = 0; i < maxInserts; i++) {
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+ tpc.proxy().updateAndFlush(userpass, testtable, mutations);
+ mutations.clear();
+ }
+ }
+
+ Key stop = new Key();
+ stop.setRow("5".getBytes());
+ ScanOptions opts = new ScanOptions();
+ opts.range = new Range(null, false, stop, false);
+ String cookie = tpc.proxy().createScanner(userpass, testtable, opts);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ i += kvList.getResultsSize();
+ hasNext = kvList.isMore();
+ }
+ assertEquals(i, 50000);
+ }
+
+ /**
+ * Insert 100000 cells which have as the row [0..99999] (padded with zeros). Filter the results so only the even numbers come back.
- *
- * @throws Exception
+ */
+ @Test
+ public void readWriteOneShotWithFilterIterator() throws Exception {
+ int maxInserts = 10000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ for (int i = 0; i < maxInserts; i++) {
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+
+ tpc.proxy().updateAndFlush(userpass, testtable, mutations);
+ mutations.clear();
+
+ }
+
+ }
+
+ String regex = ".*[02468]";
+
+ org.apache.accumulo.core.client.IteratorSetting is = new org.apache.accumulo.core.client.IteratorSetting(50, regex, RegExFilter.class);
+ RegExFilter.setRegexs(is, regex, null, null, null, false);
+
+ IteratorSetting pis = Util.iteratorSetting2ProxyIteratorSetting(is);
+ ScanOptions opts = new ScanOptions();
+ opts.iterators = Collections.singletonList(pis);
+ String cookie = tpc.proxy().createScanner(userpass, testtable, opts);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ for (KeyValue kv : kvList.getResults()) {
+ assertEquals(Integer.parseInt(new String(kv.getKey().getRow())), i);
+
+ i += 2;
+ }
+ hasNext = kvList.isMore();
+ }
+ }
+
+ // @Test
+ // This test takes kind of a long time. Enable it if you think you may have memory issues.
+ public void manyWritesAndReads() throws Exception {
+ int maxInserts = 1000000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$06d";
+ String writer = tpc.proxy().createWriter(userpass, testtable, null);
+ for (int i = 0; i < maxInserts; i++) {
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+
+ tpc.proxy().update(writer, mutations);
+ mutations.clear();
+
+ }
+
+ }
+
+ tpc.proxy().flush(writer);
+ tpc.proxy().closeWriter(writer);
+
+ String cookie = tpc.proxy().createScanner(userpass, testtable, null);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ for (KeyValue kv : kvList.getResults()) {
+ assertEquals(Integer.parseInt(new String(kv.getKey().getRow())), i);
+ i++;
+ }
+ hasNext = kvList.isMore();
+ if (hasNext)
+ assertEquals(k, kvList.getResults().size());
+ }
+ assertEquals(maxInserts, i);
+ }
+
+ @Test
+ public void asynchReadWrite() throws Exception {
+ int maxInserts = 10000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ String writer = tpc.proxy().createWriter(userpass, testtable, null);
+ for (int i = 0; i < maxInserts; i++) {
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+ tpc.proxy().update(writer, mutations);
+ mutations.clear();
+ }
+ }
+
+ tpc.proxy().flush(writer);
+ tpc.proxy().closeWriter(writer);
+
+ String regex = ".*[02468]";
+
+ org.apache.accumulo.core.client.IteratorSetting is = new org.apache.accumulo.core.client.IteratorSetting(50, regex, RegExFilter.class);
+ RegExFilter.setRegexs(is, regex, null, null, null, false);
+
+ IteratorSetting pis = Util.iteratorSetting2ProxyIteratorSetting(is);
+ ScanOptions opts = new ScanOptions();
+ opts.iterators = Collections.singletonList(pis);
+ String cookie = tpc.proxy().createScanner(userpass, testtable, opts);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ int numRead = 0;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ for (KeyValue kv : kvList.getResults()) {
+ assertEquals(i, Integer.parseInt(new String(kv.getKey().getRow())));
+ numRead++;
+ i += 2;
+ }
+ hasNext = kvList.isMore();
+ }
+ assertEquals(maxInserts / 2, numRead);
+ }
+
+ @Test
+ public void testVisibility() throws Exception {
+
+ Set<ByteBuffer> auths = new HashSet<ByteBuffer>();
+ auths.add(ByteBuffer.wrap("even".getBytes()));
+ tpc.proxy().changeUserAuthorizations(userpass, "root", auths);
+
+ int maxInserts = 10000;
+ Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
+ String format = "%1$05d";
+ String writer = tpc.proxy().createWriter(userpass, testtable, null);
+ for (int i = 0; i < maxInserts; i++) {
+ if (i % 2 == 0)
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, "even", Util.randString(10));
+ else
+ addMutation(mutations, String.format(format, i), "cf" + i, "cq" + i, "odd", Util.randString(10));
+
+ if (i % 1000 == 0 || i == maxInserts - 1) {
+ tpc.proxy().update(writer, mutations);
+ mutations.clear();
+ }
+ }
+
+ tpc.proxy().flush(writer);
+ tpc.proxy().closeWriter(writer);
+ ScanOptions opts = new ScanOptions();
+ opts.authorizations = auths;
+ String cookie = tpc.proxy().createScanner(userpass, testtable, opts);
+
+ int i = 0;
+ boolean hasNext = true;
+
+ int k = 1000;
+ int numRead = 0;
+ while (hasNext) {
+ ScanResult kvList = tpc.proxy().nextK(cookie, k);
+ for (KeyValue kv : kvList.getResults()) {
+ assertEquals(Integer.parseInt(new String(kv.getKey().getRow())), i);
+ i += 2;
+ numRead++;
+ }
+ hasNext = kvList.isMore();
+
+ }
+ assertEquals(maxInserts / 2, numRead);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java
index 442294f,0000000..05806ca
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java
+++ b/server/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java
@@@ -1,30 -1,0 +1,27 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.conf;
+
+import org.apache.accumulo.server.client.HdfsZooInstance;
+
+public class ConfigSanityCheck {
+
- /**
- * @param args
- */
+ public static void main(String[] args) {
+ new ServerConfiguration(HdfsZooInstance.getInstance()).getConfiguration();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
index 4f9d33a,0000000..01626ad
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
+++ b/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
@@@ -1,191 -1,0 +1,190 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.logger;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.tabletserver.log.DfsLogger;
+import org.apache.accumulo.server.tabletserver.log.MultiReader;
+import org.apache.accumulo.server.trace.TraceFileSystem;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+
+public class LogReader {
+
+ static class Opts extends Help {
+ @Parameter(names = "-r", description = "print only mutations associated with the given row")
+ String row;
+ @Parameter(names = "-m", description = "limit the number of mutations printed per row")
+ int maxMutations = 5;
+ @Parameter(names = "-t", description = "print only mutations that fall within the given key extent")
+ String extent;
+ @Parameter(names = "-p", description = "search for a row that matches the given regex")
+ String regexp;
+ @Parameter(description = "<logfile> { <logfile> ... }")
+ List<String> files = new ArrayList<String>();
+ }
+
+ /**
+ * Dump a Log File (Map or Sequence) to stdout. Will read from HDFS or local file system.
+ *
+ * @param args
+ * - first argument is the file to print
- * @throws IOException
+ */
+ public static void main(String[] args) throws IOException {
+ Opts opts = new Opts();
+ opts.parseArgs(LogReader.class.getName(), args);
+ Configuration conf = CachedConfiguration.getInstance();
+ FileSystem fs = TraceFileSystem.wrap(FileUtil.getFileSystem(conf, ServerConfiguration.getSiteConfiguration()));
+ FileSystem local = TraceFileSystem.wrap(FileSystem.getLocal(conf));
+
+ Matcher rowMatcher = null;
+ KeyExtent ke = null;
+ Text row = null;
+ if (opts.files.isEmpty()) {
+ new JCommander(opts).usage();
+ return;
+ }
+ if (opts.row != null)
+ row = new Text(opts.row);
+ if (opts.extent != null) {
+ String sa[] = opts.extent.split(";");
+ ke = new KeyExtent(new Text(sa[0]), new Text(sa[1]), new Text(sa[2]));
+ }
+ if (opts.regexp != null) {
+ Pattern pattern = Pattern.compile(opts.regexp);
+ rowMatcher = pattern.matcher("");
+ }
+
+ Set<Integer> tabletIds = new HashSet<Integer>();
+
+ for (String file : opts.files) {
+
+ Map<String, String> meta = new HashMap<String, String>();
+ Path path = new Path(file);
+ LogFileKey key = new LogFileKey();
+ LogFileValue value = new LogFileValue();
+
+ if (fs.isFile(path)) {
+ // read log entries from a simple hdfs file
+ FSDataInputStream f = DfsLogger.readHeader(fs, path, meta);
+ try {
+ while (true) {
+ try {
+ key.readFields(f);
+ value.readFields(f);
+ } catch (EOFException ex) {
+ break;
+ }
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
+ }
+ } finally {
+ f.close();
+ }
+ } else if (local.isFile(path)) {
+ // read log entries from a simple file
+ FSDataInputStream f = DfsLogger.readHeader(fs, path, meta);
+ try {
+ while (true) {
+ try {
+ key.readFields(f);
+ value.readFields(f);
+ } catch (EOFException ex) {
+ break;
+ }
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
+ }
+ } finally {
+ f.close();
+ }
+ } else {
+ // read the log entries sorted in a map file
+ MultiReader input = new MultiReader(fs, conf, file);
+ while (input.next(key, value)) {
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
+ }
+ }
+ }
+ }
+
+ public static void printLogEvent(LogFileKey key, LogFileValue value, Text row, Matcher rowMatcher, KeyExtent ke, Set<Integer> tabletIds, int maxMutations) {
+
+ if (ke != null) {
+ if (key.event == LogEvents.DEFINE_TABLET) {
+ if (key.tablet.equals(ke)) {
+ tabletIds.add(key.tid);
+ } else {
+ return;
+ }
+ } else if (!tabletIds.contains(key.tid)) {
+ return;
+ }
+ }
+
+ if (row != null || rowMatcher != null) {
+ if (key.event == LogEvents.MUTATION || key.event == LogEvents.MANY_MUTATIONS) {
+ boolean found = false;
+ for (Mutation m : value.mutations) {
+ if (row != null && new Text(m.getRow()).equals(row)) {
+ found = true;
+ break;
+ }
+
+ if (rowMatcher != null) {
+ rowMatcher.reset(new String(m.getRow(), Constants.UTF8));
+ if (rowMatcher.matches()) {
+ found = true;
+ break;
+ }
+ }
+ }
+
+ if (!found)
+ return;
+ } else {
+ return;
+ }
+
+ }
+
+ System.out.println(key);
+ System.out.println(LogFileValue.format(value, maxMutations));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java
index 4930bc2,0000000..e14008a
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java
@@@ -1,152 -1,0 +1,144 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.balancer;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+import java.util.SortedMap;
+
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.master.thrift.TableInfo;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletMigration;
+import org.apache.thrift.TException;
+
+/**
+ * A chaotic load balancer used for testing. It constantly shuffles tablets, preventing them from resting in a single location for very long. This is not
+ * designed for performance, do not use on production systems. I'm calling it the LokiLoadBalancer.
+ */
+public class ChaoticLoadBalancer extends TabletBalancer {
+ Random r = new Random();
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.master.balancer.TabletBalancer#getAssignments(java.util.SortedMap, java.util.Map, java.util.Map)
- */
+ @Override
+ public void getAssignments(SortedMap<TServerInstance,TabletServerStatus> current, Map<KeyExtent,TServerInstance> unassigned,
+ Map<KeyExtent,TServerInstance> assignments) {
+ long total = assignments.size() + unassigned.size();
+ long avg = (long) Math.ceil(((double) total) / current.size());
+ Map<TServerInstance,Long> toAssign = new HashMap<TServerInstance,Long>();
+ List<TServerInstance> tServerArray = new ArrayList<TServerInstance>();
+ for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
+ long numTablets = 0;
+ for (TableInfo ti : e.getValue().getTableMap().values()) {
+ numTablets += ti.tablets;
+ }
+ if (numTablets < avg) {
+ tServerArray.add(e.getKey());
+ toAssign.put(e.getKey(), avg - numTablets);
+ }
+ }
+
+ for (KeyExtent ke : unassigned.keySet())
+ {
+ int index = r.nextInt(tServerArray.size());
+ TServerInstance dest = tServerArray.get(index);
+ assignments.put(ke, dest);
+ long remaining = toAssign.get(dest).longValue() - 1;
+ if (remaining == 0) {
+ tServerArray.remove(index);
+ toAssign.remove(dest);
+ } else {
+ toAssign.put(dest, remaining);
+ }
+ }
+ }
+
+ /**
+ * Will balance randomly, maintaining distribution
+ */
+ @Override
+ public long balance(SortedMap<TServerInstance,TabletServerStatus> current, Set<KeyExtent> migrations, List<TabletMigration> migrationsOut) {
+ Map<TServerInstance,Long> numTablets = new HashMap<TServerInstance,Long>();
+ List<TServerInstance> underCapacityTServer = new ArrayList<TServerInstance>();
+
+ if (!migrations.isEmpty())
+ return 100;
+
+ boolean moveMetadata = r.nextInt(4) == 0;
+ long totalTablets = 0;
+ for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
+ long tabletCount = 0;
+ for (TableInfo ti : e.getValue().getTableMap().values()) {
+ tabletCount += ti.tablets;
+ }
+ numTablets.put(e.getKey(), tabletCount);
+ underCapacityTServer.add(e.getKey());
+ totalTablets += tabletCount;
+ }
+ // totalTablets is fuzzy due to asynchronicity of the stats
+ // *1.2 to handle fuzziness, and prevent locking for 'perfect' balancing scenarios
+ long avg = (long) Math.ceil(((double) totalTablets) / current.size() * 1.2);
+
+ for (Entry<TServerInstance, TabletServerStatus> e : current.entrySet())
+ {
+ for (String table : e.getValue().getTableMap().keySet())
+ {
+ if (!moveMetadata && "!METADATA".equals(table))
+ continue;
+ try {
+ for (TabletStats ts : getOnlineTabletsForTable(e.getKey(), table)) {
+ KeyExtent ke = new KeyExtent(ts.extent);
+ int index = r.nextInt(underCapacityTServer.size());
+ TServerInstance dest = underCapacityTServer.get(index);
+ if (dest.equals(e.getKey()))
+ continue;
+ migrationsOut.add(new TabletMigration(ke, e.getKey(), dest));
+ if (numTablets.put(dest, numTablets.get(dest) + 1) > avg)
+ underCapacityTServer.remove(index);
+ if (numTablets.put(e.getKey(), numTablets.get(e.getKey()) - 1) <= avg && !underCapacityTServer.contains(e.getKey()))
+ underCapacityTServer.add(e.getKey());
+
+ // We can get some craziness with only 1 tserver, so lets make sure there's always an option!
+ if (underCapacityTServer.isEmpty())
+ underCapacityTServer.addAll(numTablets.keySet());
+ }
+ } catch (ThriftSecurityException e1) {
+ // Shouldn't happen, but carry on if it does
+ e1.printStackTrace();
+ } catch (TException e1) {
+ // Shouldn't happen, but carry on if it does
+ e1.printStackTrace();
+ }
+ }
+ }
+
+ return 100;
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.server.master.balancer.TabletBalancer#init(org.apache.accumulo.server.conf.ServerConfiguration)
- */
+ @Override
+ public void init(ServerConfiguration conf) {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
index d6dce2f,0000000..69387d3
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
@@@ -1,150 -1,0 +1,148 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.balancer;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletMigration;
+import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransportException;
+
+public abstract class TabletBalancer {
+
+ private static final Logger log = Logger.getLogger(TabletBalancer.class);
+
+ protected ServerConfiguration configuration;
+
+ /**
+ * Initialize the TabletBalancer. This gives the balancer the opportunity to read the configuration.
+ */
+ public void init(ServerConfiguration conf) {
+ configuration = conf;
+ }
+
+ /**
+ * Assign tablets to tablet servers. This method is called whenever the master finds tablets that are unassigned.
+ *
+ * @param current
+ * The current table-summary state of all the online tablet servers. Read-only. The TabletServerStatus for each server may be null if the tablet
+ * server has not yet responded to a recent request for status.
+ * @param unassigned
+ * A map from unassigned tablet to the last known tablet server. Read-only.
+ * @param assignments
+ * A map from tablet to assigned server. Write-only.
+ */
+ abstract public void getAssignments(SortedMap<TServerInstance,TabletServerStatus> current, Map<KeyExtent,TServerInstance> unassigned,
+ Map<KeyExtent,TServerInstance> assignments);
+
+ /**
+ * Ask the balancer if any migrations are necessary.
+ *
+ * @param current
+ * The current table-summary state of all the online tablet servers. Read-only.
+ * @param migrations
+ * the current set of migrations. Read-only.
+ * @param migrationsOut
+ * new migrations to perform; should not contain tablets in the current set of migrations. Write-only.
+ * @return the time, in milliseconds, to wait before re-balancing.
+ *
+ * This method will not be called when there are unassigned tablets.
+ */
+ public abstract long balance(SortedMap<TServerInstance,TabletServerStatus> current, Set<KeyExtent> migrations, List<TabletMigration> migrationsOut);
+
+ /**
+ * Fetch the tablets for the given table by asking the tablet server. Useful if your balance strategy needs details at the tablet level to decide what tablets
+ * to move.
+ *
+ * @param tserver
+ * The tablet server to ask.
+ * @param tableId
+ * The table id
+ * @return a list of tablet statistics
+ * @throws ThriftSecurityException
+ * tablet server disapproves of your internal System password.
+ * @throws TException
+ * any other problem
+ */
+ public List<TabletStats> getOnlineTabletsForTable(TServerInstance tserver, String tableId) throws ThriftSecurityException, TException {
+ log.debug("Scanning tablet server " + tserver + " for table " + tableId);
+ Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), tserver.getLocation(), configuration.getConfiguration());
+ try {
+ List<TabletStats> onlineTabletsForTable = client.getTabletStats(Tracer.traceInfo(), SecurityConstants.getSystemCredentials(), tableId);
+ return onlineTabletsForTable;
+ } catch (TTransportException e) {
+ log.error("Unable to connect to " + tserver + ": " + e);
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ return null;
+ }
+
+ /**
+ * Utility to ensure that the migrations from balance() are consistent:
+ * <ul>
+ * <li>Tablet objects are not null
+ * <li>Source and destination tablet servers are not null and current
+ * </ul>
+ *
- * @param current
- * @param migrations
+ * @return A list of TabletMigration object that passed sanity checks.
+ */
+ public static List<TabletMigration> checkMigrationSanity(Set<TServerInstance> current, List<TabletMigration> migrations) {
+ List<TabletMigration> result = new ArrayList<TabletMigration>(migrations.size());
+ for (TabletMigration m : migrations) {
+ if (m.tablet == null) {
+ log.warn("Balancer gave back a null tablet " + m);
+ continue;
+ }
+ if (m.newServer == null) {
+ log.warn("Balancer did not set the destination " + m);
+ continue;
+ }
+ if (m.oldServer == null) {
+ log.warn("Balancer did not set the source " + m);
+ continue;
+ }
+ if (!current.contains(m.oldServer)) {
+ log.warn("Balancer wants to move a tablet from a server that is not current: " + m);
+ continue;
+ }
+ if (!current.contains(m.newServer)) {
+ log.warn("Balancer wants to move a tablet to a server that is not current: " + m);
+ continue;
+ }
+ result.add(m);
+ }
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
index f9f03bd,0000000..540ebc0
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
@@@ -1,87 -1,0 +1,81 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.state;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+
+/**
+ * Interface for storing information about tablet assignments. There are three implementations:
+ *
+ * ZooTabletStateStore: information about the root tablet is stored in ZooKeeper MetaDataStateStore: information about the other tablets are stored in the
+ * metadata table
+ *
+ */
+public abstract class TabletStateStore implements Iterable<TabletLocationState> {
+
+ /**
+ * Identifying name for this tablet state store.
+ */
+ abstract public String name();
+
+ /**
+ * Scan the information about the tablets covered by this store
+ */
++ @Override
+ abstract public Iterator<TabletLocationState> iterator();
+
+ /**
+ * Store the assigned locations in the data store.
- *
- * @param assignments
- * @throws DistributedStoreException
+ */
+ abstract public void setFutureLocations(Collection<Assignment> assignments) throws DistributedStoreException;
+
+ /**
+ * Tablet servers will update the data store with the location when they bring the tablet online
- *
- * @param assignments
- * @throws DistributedStoreException
+ */
+ abstract public void setLocations(Collection<Assignment> assignments) throws DistributedStoreException;
+
+ /**
+ * Mark the tablets as having no known or future location.
+ *
+ * @param tablets
+ * the tablets' current information
- * @throws DistributedStoreException
+ */
+ abstract public void unassign(Collection<TabletLocationState> tablets) throws DistributedStoreException;
+
+ public static void unassign(TabletLocationState tls) throws DistributedStoreException {
+ TabletStateStore store;
+ if (tls.extent.isRootTablet()) {
+ store = new ZooTabletStateStore();
+ } else {
+ store = new MetaDataStateStore();
+ }
+ store.unassign(Collections.singletonList(tls));
+ }
+
+ public static void setLocation(Assignment assignment) throws DistributedStoreException {
+ TabletStateStore store;
+ if (assignment.tablet.isRootTablet()) {
+ store = new ZooTabletStateStore();
+ } else {
+ store = new MetaDataStateStore();
+ }
+ store.setLocations(Collections.singletonList(assignment));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/master/tableOps/TraceRepo.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/master/tableOps/TraceRepo.java
index 58a337f,0000000..45f6a60
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/master/tableOps/TraceRepo.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/tableOps/TraceRepo.java
@@@ -1,109 -1,0 +1,84 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.tableOps;
+
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.instrument.Trace;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.accumulo.trace.thrift.TInfo;
+import org.apache.accumulo.fate.Repo;
+
+
+/**
+ *
+ */
+public class TraceRepo<T> implements Repo<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ TInfo tinfo;
+ Repo<T> repo;
+
+ public TraceRepo(Repo<T> repo) {
+ this.repo = repo;
+ tinfo = Tracer.traceInfo();
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.server.fate.Repo#isReady(long, java.lang.Object)
- */
+ @Override
+ public long isReady(long tid, T environment) throws Exception {
+ Span span = Trace.trace(tinfo, repo.getDescription());
+ try {
+ return repo.isReady(tid, environment);
+ } finally {
+ span.stop();
+ }
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.server.fate.Repo#call(long, java.lang.Object)
- */
+ @Override
+ public Repo<T> call(long tid, T environment) throws Exception {
+ Span span = Trace.trace(tinfo, repo.getDescription());
+ try {
+ Repo<T> result = repo.call(tid, environment);
+ if (result == null)
+ return result;
+ return new TraceRepo<T>(result);
+ } finally {
+ span.stop();
+ }
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.server.fate.Repo#undo(long, java.lang.Object)
- */
+ @Override
+ public void undo(long tid, T environment) throws Exception {
+ Span span = Trace.trace(tinfo, repo.getDescription());
+ try {
+ repo.undo(tid, environment);
+ } finally {
+ span.stop();
+ }
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.server.fate.Repo#getDescription()
- */
+ @Override
+ public String getDescription() {
+ return repo.getDescription();
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.server.fate.Repo#getReturn()
- */
+ @Override
+ public String getReturn() {
+ return repo.getReturn();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/metanalysis/LogFileOutputFormat.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/metanalysis/LogFileOutputFormat.java
index 829d7bc,0000000..7e50754
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/metanalysis/LogFileOutputFormat.java
+++ b/server/src/main/java/org/apache/accumulo/server/metanalysis/LogFileOutputFormat.java
@@@ -1,70 -1,0 +1,66 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.metanalysis;
+
+import java.io.IOException;
+
+import org.apache.accumulo.server.logger.LogFileKey;
+import org.apache.accumulo.server.logger.LogFileValue;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+
+/**
+ * Output format for Accumulo write ahead logs.
+ */
+public class LogFileOutputFormat extends FileOutputFormat<LogFileKey,LogFileValue> {
+
+ private static class LogFileRecordWriter extends RecordWriter<LogFileKey,LogFileValue> {
+
+ private FSDataOutputStream out;
+
- /**
- * @param outputPath
- * @throws IOException
- */
+ public LogFileRecordWriter(Path outputPath) throws IOException {
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+
+ out = fs.create(outputPath);
+ }
+
+ @Override
+ public void close(TaskAttemptContext arg0) throws IOException, InterruptedException {
+ out.close();
+ }
+
+ @Override
+ public void write(LogFileKey key, LogFileValue val) throws IOException, InterruptedException {
+ key.write(out);
+ val.write(out);
+ }
+
+ }
+
+ @Override
+ public RecordWriter<LogFileKey,LogFileValue> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
+ Path outputPath = getDefaultWorkFile(context, "");
+ return new LogFileRecordWriter(outputPath);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/metanalysis/PrintEvents.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/metanalysis/PrintEvents.java
index 88f5cbe,0000000..0478d83
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/metanalysis/PrintEvents.java
+++ b/server/src/main/java/org/apache/accumulo/server/metanalysis/PrintEvents.java
@@@ -1,109 -1,0 +1,106 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.metanalysis;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.Constants;
- import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
++import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.server.logger.LogFileValue;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Looks up and prints mutations indexed by IndexMeta
+ */
+public class PrintEvents {
+
+ static class Opts extends ClientOpts {
+ @Parameter(names={"-t", "--tableId"}, description="table id", required=true)
+ String tableId;
+ @Parameter(names={"-e", "--endRow"}, description="end row")
+ String endRow;
+ @Parameter(names={"-t", "--time"}, description="time, in milliseconds", required=true)
+ long time;
+ }
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(PrintEvents.class.getName(), args);
+
+ Connector conn = opts.getConnector();
+
+ printEvents(conn, opts.tableId, opts.endRow, opts.time);
+ }
+
+ /**
+ * @param conn
+ * @param tablePrefix
+ * @param tableId
+ * @param endRow
+ * @param time
+ */
+ private static void printEvents(Connector conn, String tableId, String endRow, Long time) throws Exception {
+ Scanner scanner = conn.createScanner("tabletEvents", new Authorizations());
+ String metaRow = tableId + (endRow == null ? "<" : ";" + endRow);
+ scanner.setRange(new Range(new Key(metaRow, String.format("%020d", time)), true, new Key(metaRow).followingKey(PartialKey.ROW), false));
+ int count = 0;
+
+ String lastLog = null;
+
+ loop1: for (Entry<Key,Value> entry : scanner) {
+ if (entry.getKey().getColumnQualifier().toString().equals("log")) {
+ if (lastLog == null || !lastLog.equals(entry.getValue().toString()))
+ System.out.println("Log : " + entry.getValue());
+ lastLog = entry.getValue().toString();
+ } else if (entry.getKey().getColumnQualifier().toString().equals("mut")) {
+ DataInputStream dis = new DataInputStream(new ByteArrayInputStream(entry.getValue().get()));
+ Mutation m = new Mutation();
+ m.readFields(dis);
+
+ LogFileValue lfv = new LogFileValue();
+ lfv.mutations = Collections.singletonList(m);
+
+ System.out.println(LogFileValue.format(lfv, 1));
+
+ List<ColumnUpdate> columnsUpdates = m.getUpdates();
+ for (ColumnUpdate cu : columnsUpdates) {
+ if (Constants.METADATA_PREV_ROW_COLUMN.equals(new Text(cu.getColumnFamily()), new Text(cu.getColumnQualifier())) && count > 0) {
+ System.out.println("Saw change to prevrow, stopping printing events.");
+ break loop1;
+ }
+ }
+ count++;
+ }
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
index 5a8ddec,0000000..d76c7a3
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
+++ b/server/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
@@@ -1,277 -1,0 +1,273 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.metrics;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.lang.management.ManagementFactory;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+import javax.management.StandardMBean;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.commons.lang.builder.ToStringBuilder;
+import org.apache.commons.lang.time.DateUtils;
+
+public abstract class AbstractMetricsImpl {
+
+ public class Metric {
+
+ private long count = 0;
+ private long avg = 0;
+ private long min = 0;
+ private long max = 0;
+
+ public long getCount() {
+ return count;
+ }
+
+ public long getAvg() {
+ return avg;
+ }
+
+ public long getMin() {
+ return min;
+ }
+
+ public long getMax() {
+ return max;
+ }
+
+ public void incCount() {
+ count++;
+ }
+
+ public void addAvg(long a) {
+ if (a < 0)
+ return;
+ avg = (long) ((avg * .8) + (a * .2));
+ }
+
+ public void addMin(long a) {
+ if (a < 0)
+ return;
+ min = Math.min(min, a);
+ }
+
+ public void addMax(long a) {
+ if (a < 0)
+ return;
+ max = Math.max(max, a);
+ }
+
+ @Override
+ public String toString() {
+ return new ToStringBuilder(this).append("count", count).append("average", avg).append("minimum", min).append("maximum", max).toString();
+ }
+
+ }
+
+ static final org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger(AbstractMetricsImpl.class);
+
+ private static ConcurrentHashMap<String,Metric> registry = new ConcurrentHashMap<String,Metric>();
+
+ private boolean currentlyLogging = false;
+
+ private File logDir = null;
+
+ private String metricsPrefix = null;
+
+ private Date today = new Date();
+
+ private File logFile = null;
+
+ private Writer logWriter = null;
+
+ private SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd");
+
+ private SimpleDateFormat logFormatter = new SimpleDateFormat("yyyyMMddhhmmssz");
+
+ private MetricsConfiguration config = null;
+
+ public AbstractMetricsImpl() {
+ this.metricsPrefix = getMetricsPrefix();
+ config = new MetricsConfiguration(metricsPrefix);
+ }
+
+ /**
+ * Registers a StandardMBean with the MBean Server
- *
- * @throws Exception
+ */
+ public void register(StandardMBean mbean) throws Exception {
+ // Register this object with the MBeanServer
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ if (null == getObjectName())
+ throw new IllegalArgumentException("MBean object name must be set.");
+ mbs.registerMBean(mbean, getObjectName());
+
+ setupLogging();
+ }
+
+ /**
+ * Registers this MBean with the MBean Server
- *
- * @throws Exception
+ */
+ public void register() throws Exception {
+ // Register this object with the MBeanServer
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ if (null == getObjectName())
+ throw new IllegalArgumentException("MBean object name must be set.");
+ mbs.registerMBean(this, getObjectName());
+
+ setupLogging();
+ }
+
+ public void createMetric(String name) {
+ registry.put(name, new Metric());
+ }
+
+ public Metric getMetric(String name) {
+ return registry.get(name);
+ }
+
+ public long getMetricCount(String name) {
+ return registry.get(name).getCount();
+ }
+
+ public long getMetricAvg(String name) {
+ return registry.get(name).getAvg();
+ }
+
+ public long getMetricMin(String name) {
+ return registry.get(name).getMin();
+ }
+
+ public long getMetricMax(String name) {
+ return registry.get(name).getMax();
+ }
+
+ private void setupLogging() throws IOException {
+ if (null == config.getMetricsConfiguration())
+ return;
+ // If we are already logging, then return
+ if (!currentlyLogging && config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ // Check to see if directory exists, else make it
+ String mDir = config.getMetricsConfiguration().getString("logging.dir");
+ if (null != mDir) {
+ File dir = new File(mDir);
+ if (!dir.isDirectory())
+ if (!dir.mkdir())
+ log.warn("Could not create log directory: " + dir);
+ logDir = dir;
+ // Create new log file
+ startNewLog();
+ }
+ currentlyLogging = true;
+ }
+ }
+
+ private void startNewLog() throws IOException {
+ if (null != logWriter) {
+ logWriter.flush();
+ logWriter.close();
+ }
+ logFile = new File(logDir, metricsPrefix + "-" + formatter.format(today) + ".log");
+ if (!logFile.exists()) {
+ if (!logFile.createNewFile()) {
+ log.error("Unable to create new log file");
+ currentlyLogging = false;
+ return;
+ }
+ }
+ logWriter = new OutputStreamWriter(new FileOutputStream(logFile, true), Constants.UTF8);
+ }
+
+ private void writeToLog(String name) throws IOException {
+ if (null == logWriter)
+ return;
+ // Increment the date if we have to
+ Date now = new Date();
+ if (!DateUtils.isSameDay(today, now)) {
+ today = now;
+ startNewLog();
+ }
+ logWriter.append(logFormatter.format(now)).append(" Metric: ").append(name).append(": ").append(registry.get(name).toString()).append("\n");
+ }
+
+ public void add(String name, long time) {
+ if (isEnabled()) {
+ registry.get(name).incCount();
+ registry.get(name).addAvg(time);
+ registry.get(name).addMin(time);
+ registry.get(name).addMax(time);
+ // If we are not currently logging and should be, then initialize
+ if (!currentlyLogging && config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ try {
+ setupLogging();
+ } catch (IOException ioe) {
+ log.error("Error setting up log", ioe);
+ }
+ } else if (currentlyLogging && !config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ // if we are currently logging and shouldn't be, then close logs
+ try {
+ logWriter.flush();
+ logWriter.close();
+ logWriter = null;
+ logFile = null;
+ } catch (Exception e) {
+ log.error("Error stopping metrics logging", e);
+ }
+ currentlyLogging = false;
+ }
+ if (currentlyLogging) {
+ try {
+ writeToLog(name);
+ } catch (IOException ioe) {
+ log.error("Error writing to metrics log", ioe);
+ }
+ }
+ }
+ }
+
+ public boolean isEnabled() {
+ return config.isEnabled();
+ }
+
+ protected abstract ObjectName getObjectName();
+
+ protected abstract String getMetricsPrefix();
+
+ @Override
+ protected void finalize() {
+ if (null != logWriter) {
+ try {
+ logWriter.close();
+ } catch (Exception e) {
+ // do nothing
+ } finally {
+ logWriter = null;
+ }
+ }
+ logFile = null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/security/handler/InsecurePermHandler.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/security/handler/InsecurePermHandler.java
index d51f3f9,0000000..3bda9d6
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/security/handler/InsecurePermHandler.java
+++ b/server/src/main/java/org/apache/accumulo/server/security/handler/InsecurePermHandler.java
@@@ -1,146 -1,0 +1,104 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security.handler;
+
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+
+/**
+ * This is a Permission Handler implementation that doesn't actually do any security. Use at your own risk.
+ */
+public class InsecurePermHandler implements PermissionHandler {
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#initialize(java.lang.String)
- */
+ @Override
+ public void initialize(String instanceId, boolean initialize) {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#validSecurityHandlers(org.apache.accumulo.server.security.handler.Authenticator, org.apache.accumulo.server.security.handler.Authorizor)
- */
+ @Override
+ public boolean validSecurityHandlers(Authenticator authent, Authorizor author) {
+ return true;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#initializeSecurity(java.lang.String)
- */
+ @Override
+ public void initializeSecurity(TCredentials token, String rootuser) throws AccumuloSecurityException {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#hasSystemPermission(java.lang.String, org.apache.accumulo.core.security.SystemPermission)
- */
+ @Override
+ public boolean hasSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ return true;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#hasCachedSystemPermission(java.lang.String, org.apache.accumulo.core.security.SystemPermission)
- */
+ @Override
+ public boolean hasCachedSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ return true;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#hasTablePermission(java.lang.String, java.lang.String, org.apache.accumulo.core.security.TablePermission)
- */
+ @Override
+ public boolean hasTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ return true;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#hasCachedTablePermission(java.lang.String, java.lang.String, org.apache.accumulo.core.security.TablePermission)
- */
+ @Override
+ public boolean hasCachedTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ return true;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#grantSystemPermission(java.lang.String, org.apache.accumulo.core.security.SystemPermission)
- */
+ @Override
+ public void grantSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#revokeSystemPermission(java.lang.String, org.apache.accumulo.core.security.SystemPermission)
- */
+ @Override
+ public void revokeSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#grantTablePermission(java.lang.String, java.lang.String, org.apache.accumulo.core.security.TablePermission)
- */
+ @Override
+ public void grantTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#revokeTablePermission(java.lang.String, java.lang.String, org.apache.accumulo.core.security.TablePermission)
- */
+ @Override
+ public void revokeTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#cleanTablePermissions(java.lang.String)
- */
+ @Override
+ public void cleanTablePermissions(String table) throws AccumuloSecurityException, TableNotFoundException {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#initUser(java.lang.String)
- */
+ @Override
+ public void initUser(String user) throws AccumuloSecurityException {
+ return;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.server.security.handler.PermissionHandler#dropUser(java.lang.String)
- */
+ @Override
+ public void cleanUser(String user) throws AccumuloSecurityException {
+ return;
+ }
+
+ @Override
+ public void initTable(String table) throws AccumuloSecurityException {
+ }
+
+}
[15/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
index 448981b,0000000..442f1be
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
@@@ -1,1322 -1,0 +1,1318 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.admin;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableDeletedException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.client.impl.AccumuloServerException;
+import org.apache.accumulo.core.client.impl.ClientExec;
+import org.apache.accumulo.core.client.impl.ClientExecReturn;
+import org.apache.accumulo.core.client.impl.MasterClient;
+import org.apache.accumulo.core.client.impl.ServerClient;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.TabletLocator;
+import org.apache.accumulo.core.client.impl.TabletLocator.TabletLocation;
+import org.apache.accumulo.core.client.impl.thrift.ClientService;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.constraints.Constraint;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.master.thrift.TableOperation;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.LocalityGroupUtil;
+import org.apache.accumulo.core.util.MetadataTable;
+import org.apache.accumulo.core.util.NamingThreadFactory;
+import org.apache.accumulo.core.util.OpTimer;
+import org.apache.accumulo.core.util.StringUtil;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TApplicationException;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransportException;
+
+/**
+ * Provides a class for administering tables
+ *
+ */
+public class TableOperationsImpl extends TableOperationsHelper {
+ private Instance instance;
+ private TCredentials credentials;
+
+ public static final String CLONE_EXCLUDE_PREFIX = "!";
+
+ private static final Logger log = Logger.getLogger(TableOperations.class);
+
+ /**
+ * @param instance
+ * the connection information for this instance
+ * @param credentials
+ * the username/password for this connection
+ */
+ public TableOperationsImpl(Instance instance, TCredentials credentials) {
+ ArgumentChecker.notNull(instance, credentials);
+ this.instance = instance;
+ this.credentials = credentials;
+ }
+
+ /**
+ * Retrieve a list of tables in Accumulo.
+ *
+ * @return List of tables in accumulo
+ */
+ @Override
+ public SortedSet<String> list() {
+ OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Fetching list of tables...");
+ TreeSet<String> tableNames = new TreeSet<String>(Tables.getNameToIdMap(instance).keySet());
+ opTimer.stop("Fetched " + tableNames.size() + " table names in %DURATION%");
+ return tableNames;
+ }
+
+ /**
+ * A method to check if a table exists in Accumulo.
+ *
+ * @param tableName
+ * the name of the table
+ * @return true if the table exists
+ */
+ @Override
+ public boolean exists(String tableName) {
+ ArgumentChecker.notNull(tableName);
+ if (tableName.equals(Constants.METADATA_TABLE_NAME))
+ return true;
+
+ OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Checking if table " + tableName + "exists...");
+ boolean exists = Tables.getNameToIdMap(instance).containsKey(tableName);
+ opTimer.stop("Checked existance of " + exists + " in %DURATION%");
+ return exists;
+ }
+
+ /**
+ * Create a table with no special configuration
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableExistsException
+ * if the table already exists
+ */
+ @Override
+ public void create(String tableName) throws AccumuloException, AccumuloSecurityException, TableExistsException {
+ create(tableName, true, TimeType.MILLIS);
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param limitVersion
+ * Enables/disables the versioning iterator, which will limit the number of Key versions kept.
+ */
+ @Override
+ public void create(String tableName, boolean limitVersion) throws AccumuloException, AccumuloSecurityException, TableExistsException {
+ create(tableName, limitVersion, TimeType.MILLIS);
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param timeType
+ * specifies logical or real-time based time recording for entries in the table
+ * @param limitVersion
+ * Enables/disables the versioning iterator, which will limit the number of Key versions kept.
+ */
+ @Override
+ public void create(String tableName, boolean limitVersion, TimeType timeType) throws AccumuloException, AccumuloSecurityException, TableExistsException {
+ ArgumentChecker.notNull(tableName, timeType);
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(timeType.name().getBytes(Constants.UTF8)));
+
+ Map<String,String> opts = IteratorUtil.generateInitialTableProperties(limitVersion);
+
+ try {
+ doTableOperation(TableOperation.CREATE, args, opts);
+ } catch (TableNotFoundException e1) {
+ // should not happen
+ throw new RuntimeException(e1);
+ }
+ }
+
+ private long beginTableOperation() throws ThriftSecurityException, TException {
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ return client.beginTableOperation(Tracer.traceInfo(), credentials);
+ } catch (TTransportException tte) {
+ log.debug("Failed to call beginTableOperation(), retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ }
+
+ private void executeTableOperation(long opid, TableOperation op, List<ByteBuffer> args, Map<String,String> opts, boolean autoCleanUp)
+ throws ThriftSecurityException, TException, ThriftTableOperationException {
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ client.executeTableOperation(Tracer.traceInfo(), credentials, opid, op, args, opts, autoCleanUp);
+ break;
+ } catch (TTransportException tte) {
+ log.debug("Failed to call executeTableOperation(), retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ }
+
+ private String waitForTableOperation(long opid) throws ThriftSecurityException, TException, ThriftTableOperationException {
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ return client.waitForTableOperation(Tracer.traceInfo(), credentials, opid);
+ } catch (TTransportException tte) {
+ log.debug("Failed to call waitForTableOperation(), retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ }
+
+ private void finishTableOperation(long opid) throws ThriftSecurityException, TException {
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ client.finishTableOperation(Tracer.traceInfo(), credentials, opid);
+ break;
+ } catch (TTransportException tte) {
+ log.debug("Failed to call finishTableOperation(), retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ }
+
+ private String doTableOperation(TableOperation op, List<ByteBuffer> args, Map<String,String> opts) throws AccumuloSecurityException, TableExistsException,
+ TableNotFoundException, AccumuloException {
+ return doTableOperation(op, args, opts, true);
+ }
+
+ private String doTableOperation(TableOperation op, List<ByteBuffer> args, Map<String,String> opts, boolean wait) throws AccumuloSecurityException,
+ TableExistsException, TableNotFoundException, AccumuloException {
+ Long opid = null;
+
+ try {
+ opid = beginTableOperation();
+ executeTableOperation(opid, op, args, opts, !wait);
+ if (!wait) {
+ opid = null;
+ return null;
+ }
+ String ret = waitForTableOperation(opid);
+ Tables.clearCache(instance);
+ return ret;
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (ThriftTableOperationException e) {
+ switch (e.getType()) {
+ case EXISTS:
+ throw new TableExistsException(e);
+ case NOTFOUND:
+ throw new TableNotFoundException(e);
+ case OFFLINE:
+ throw new TableOfflineException(instance, null);
+ case OTHER:
+ default:
+ throw new AccumuloException(e.description, e);
+ }
+ } catch (Exception e) {
+ throw new AccumuloException(e.getMessage(), e);
+ } finally {
+ // always finish table op, even when exception
+ if (opid != null)
+ try {
+ finishTableOperation(opid);
+ } catch (Exception e) {
+ log.warn(e.getMessage(), e);
+ }
+ }
+ }
+
+ private static class SplitEnv {
+ private String tableName;
+ private String tableId;
+ private ExecutorService executor;
+ private CountDownLatch latch;
+ private AtomicReference<Exception> exception;
+
+ SplitEnv(String tableName, String tableId, ExecutorService executor, CountDownLatch latch, AtomicReference<Exception> exception) {
+ this.tableName = tableName;
+ this.tableId = tableId;
+ this.executor = executor;
+ this.latch = latch;
+ this.exception = exception;
+ }
+ }
+
+ private class SplitTask implements Runnable {
+
+ private List<Text> splits;
+ private SplitEnv env;
+
+ SplitTask(SplitEnv env, List<Text> splits) {
+ this.env = env;
+ this.splits = splits;
+ }
+
+ @Override
+ public void run() {
+ try {
+ if (env.exception.get() != null)
+ return;
+
+ if (splits.size() <= 2) {
+ addSplits(env.tableName, new TreeSet<Text>(splits), env.tableId);
+ for (int i = 0; i < splits.size(); i++)
+ env.latch.countDown();
+ return;
+ }
+
+ int mid = splits.size() / 2;
+
+ // split the middle split point to ensure that child task split different tablets and can therefore
+ // run in parallel
+ addSplits(env.tableName, new TreeSet<Text>(splits.subList(mid, mid + 1)), env.tableId);
+ env.latch.countDown();
+
+ env.executor.submit(new SplitTask(env, splits.subList(0, mid)));
+ env.executor.submit(new SplitTask(env, splits.subList(mid + 1, splits.size())));
+
+ } catch (Exception e) {
+ env.exception.compareAndSet(null, e);
+ }
+ }
+
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param partitionKeys
+ * a sorted set of row key values to pre-split the table on
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public void addSplits(String tableName, SortedSet<Text> partitionKeys) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ String tableId = Tables.getTableId(instance, tableName);
+
+ List<Text> splits = new ArrayList<Text>(partitionKeys);
+ // should be sorted because we copied from a sorted set, but that makes assumptions about
+ // how the copy was done so resort to be sure.
+ Collections.sort(splits);
+
+ CountDownLatch latch = new CountDownLatch(splits.size());
+ AtomicReference<Exception> exception = new AtomicReference<Exception>(null);
+
+ ExecutorService executor = Executors.newFixedThreadPool(16, new NamingThreadFactory("addSplits"));
+ try {
+ executor.submit(new SplitTask(new SplitEnv(tableName, tableId, executor, latch, exception), splits));
+
+ while (!latch.await(100, TimeUnit.MILLISECONDS)) {
+ if (exception.get() != null) {
+ executor.shutdownNow();
+ Exception excep = exception.get();
+ if (excep instanceof TableNotFoundException)
+ throw (TableNotFoundException) excep;
+ else if (excep instanceof AccumuloException)
+ throw (AccumuloException) excep;
+ else if (excep instanceof AccumuloSecurityException)
+ throw (AccumuloSecurityException) excep;
+ else if (excep instanceof RuntimeException)
+ throw (RuntimeException) excep;
+ else
+ throw new RuntimeException(excep);
+ }
+ }
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ } finally {
+ executor.shutdown();
+ }
+ }
+
+ private void addSplits(String tableName, SortedSet<Text> partitionKeys, String tableId) throws AccumuloException, AccumuloSecurityException,
+ TableNotFoundException, AccumuloServerException {
+ TabletLocator tabLocator = TabletLocator.getInstance(instance, new Text(tableId));
+
+ for (Text split : partitionKeys) {
+ boolean successful = false;
+ int attempt = 0;
+
+ while (!successful) {
+
+ if (attempt > 0)
+ UtilWaitThread.sleep(100);
+
+ attempt++;
+
+ TabletLocation tl = tabLocator.locateTablet(split, false, false, credentials);
+
+ if (tl == null) {
+ if (!Tables.exists(instance, tableId))
+ throw new TableNotFoundException(tableId, tableName, null);
+ else if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
+ throw new TableOfflineException(instance, tableId);
+ continue;
+ }
+
+ try {
+ TabletClientService.Client client = ThriftUtil.getTServerClient(tl.tablet_location, instance.getConfiguration());
+ try {
+ OpTimer opTimer = null;
+ if (log.isTraceEnabled())
+ opTimer = new OpTimer(log, Level.TRACE).start("Splitting tablet " + tl.tablet_extent + " on " + tl.tablet_location + " at " + split);
+
+ client.splitTablet(Tracer.traceInfo(), credentials, tl.tablet_extent.toThrift(), TextUtil.getByteBuffer(split));
+
+ // just split it, might as well invalidate it in the cache
+ tabLocator.invalidateCache(tl.tablet_extent);
+
+ if (opTimer != null)
+ opTimer.stop("Split tablet in %DURATION%");
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+
+ } catch (TApplicationException tae) {
+ throw new AccumuloServerException(tl.tablet_location, tae);
+ } catch (TTransportException e) {
+ tabLocator.invalidateCache(tl.tablet_location);
+ continue;
+ } catch (ThriftSecurityException e) {
+ Tables.clearCache(instance);
+ if (!Tables.exists(instance, tableId))
+ throw new TableNotFoundException(tableId, tableName, null);
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (NotServingTabletException e) {
+ tabLocator.invalidateCache(tl.tablet_extent);
+ continue;
+ } catch (TException e) {
+ tabLocator.invalidateCache(tl.tablet_location);
+ continue;
+ }
+
+ successful = true;
+ }
+ }
+ }
+
+ @Override
+ public void merge(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+
+ ArgumentChecker.notNull(tableName);
+ ByteBuffer EMPTY = ByteBuffer.allocate(0);
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), start == null ? EMPTY : TextUtil.getByteBuffer(start), end == null ? EMPTY
+ : TextUtil.getByteBuffer(end));
+ Map<String,String> opts = new HashMap<String,String>();
+ try {
+ doTableOperation(TableOperation.MERGE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void deleteRows(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+
+ ArgumentChecker.notNull(tableName);
+ ByteBuffer EMPTY = ByteBuffer.allocate(0);
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), start == null ? EMPTY : TextUtil.getByteBuffer(start), end == null ? EMPTY
+ : TextUtil.getByteBuffer(end));
+ Map<String,String> opts = new HashMap<String,String>();
+ try {
+ doTableOperation(TableOperation.DELETE_RANGE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @return the split points (end-row names) for the table's current split profile
+ */
+ @Override
+ public Collection<Text> listSplits(String tableName) throws TableNotFoundException, AccumuloSecurityException {
+
+ ArgumentChecker.notNull(tableName);
+
+ String tableId = Tables.getTableId(instance, tableName);
+
+ SortedSet<KeyExtent> tablets = new TreeSet<KeyExtent>();
+ Map<KeyExtent,String> locations = new TreeMap<KeyExtent,String>();
+
+ while (true) {
+ try {
+ tablets.clear();
+ locations.clear();
+ // the following method throws AccumuloException for some conditions that should be retried
+ MetadataTable.getEntries(instance, credentials, tableId, true, locations, tablets);
+ break;
+ } catch (AccumuloSecurityException ase) {
+ throw ase;
+ } catch (Throwable t) {
+ if (!Tables.exists(instance, tableId)) {
+ throw new TableNotFoundException(tableId, tableName, null);
+ }
+
+ if (t instanceof RuntimeException && t.getCause() instanceof AccumuloSecurityException) {
+ throw (AccumuloSecurityException) t.getCause();
+ }
+
+ log.info(t.getMessage() + " ... retrying ...");
+ UtilWaitThread.sleep(3000);
+ }
+ }
+
+ ArrayList<Text> endRows = new ArrayList<Text>(tablets.size());
+
+ for (KeyExtent ke : tablets)
+ if (ke.getEndRow() != null)
+ endRows.add(ke.getEndRow());
+
+ return endRows;
+ }
+
+ @Deprecated
+ @Override
+ public Collection<Text> getSplits(String tableName) throws TableNotFoundException {
+ try {
+ return listSplits(tableName);
+ } catch (AccumuloSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param maxSplits
+ * specifies the maximum number of splits to return
+ * @return the split points (end-row names) for the table's current split profile, grouped into fewer splits so as not to exceed maxSplits
- * @throws TableNotFoundException
+ */
+ @Override
+ public Collection<Text> listSplits(String tableName, int maxSplits) throws TableNotFoundException, AccumuloSecurityException {
+ Collection<Text> endRows = listSplits(tableName);
+
+ if (endRows.size() <= maxSplits)
+ return endRows;
+
+ double r = (maxSplits + 1) / (double) (endRows.size());
+ double pos = 0;
+
+ ArrayList<Text> subset = new ArrayList<Text>(maxSplits);
+
+ int j = 0;
+ for (int i = 0; i < endRows.size() && j < maxSplits; i++) {
+ pos += r;
+ while (pos > 1) {
+ subset.add(((ArrayList<Text>) endRows).get(i));
+ j++;
+ pos -= 1;
+ }
+ }
+
+ return subset;
+ }
+
+ @Deprecated
+ @Override
+ public Collection<Text> getSplits(String tableName, int maxSplits) throws TableNotFoundException {
+ try {
+ return listSplits(tableName, maxSplits);
+ } catch (AccumuloSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Delete a table
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public void delete(String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)));
+ Map<String,String> opts = new HashMap<String,String>();
+
+ try {
+ doTableOperation(TableOperation.DELETE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ @Override
+ public void clone(String srcTableName, String newTableName, boolean flush, Map<String,String> propertiesToSet, Set<String> propertiesToExclude)
+ throws AccumuloSecurityException, TableNotFoundException, AccumuloException, TableExistsException {
+
+ ArgumentChecker.notNull(srcTableName, newTableName);
+
+ String srcTableId = Tables.getTableId(instance, srcTableName);
+
+ if (flush)
+ _flush(srcTableId, null, null, true);
+
+ if (propertiesToExclude == null)
+ propertiesToExclude = Collections.emptySet();
+
+ if (propertiesToSet == null)
+ propertiesToSet = Collections.emptyMap();
+
+ if (!Collections.disjoint(propertiesToExclude, propertiesToSet.keySet()))
+ throw new IllegalArgumentException("propertiesToSet and propertiesToExclude not disjoint");
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(srcTableId.getBytes(Constants.UTF8)), ByteBuffer.wrap(newTableName.getBytes(Constants.UTF8)));
+ Map<String,String> opts = new HashMap<String,String>();
+ for (Entry<String,String> entry : propertiesToSet.entrySet()) {
+ if (entry.getKey().startsWith(CLONE_EXCLUDE_PREFIX))
+ throw new IllegalArgumentException("Property can not start with " + CLONE_EXCLUDE_PREFIX);
+ opts.put(entry.getKey(), entry.getValue());
+ }
+
+ for (String prop : propertiesToExclude) {
+ opts.put(CLONE_EXCLUDE_PREFIX + prop, "");
+ }
+
+ doTableOperation(TableOperation.CLONE, args, opts);
+ }
+
+ /**
+ * Rename a table
+ *
+ * @param oldTableName
+ * the old table name
+ * @param newTableName
+ * the new table name
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the old table name does not exist
+ * @throws TableExistsException
+ * if the new table name already exists
+ */
+ @Override
+ public void rename(String oldTableName, String newTableName) throws AccumuloSecurityException, TableNotFoundException, AccumuloException,
+ TableExistsException {
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(oldTableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(newTableName.getBytes(Constants.UTF8)));
+ Map<String,String> opts = new HashMap<String,String>();
+ doTableOperation(TableOperation.RENAME, args, opts);
+ }
+
+ /**
+ * @deprecated since 1.4 {@link #flush(String, Text, Text, boolean)}
+ */
+ @Override
+ @Deprecated
+ public void flush(String tableName) throws AccumuloException, AccumuloSecurityException {
+ try {
+ flush(tableName, null, null, false);
+ } catch (TableNotFoundException e) {
+ throw new AccumuloException(e.getMessage(), e);
+ }
+ }
+
+ /**
+ * Flush a table
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
- * @throws TableNotFoundException
+ */
+ @Override
+ public void flush(String tableName, Text start, Text end, boolean wait) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+
+ String tableId = Tables.getTableId(instance, tableName);
+ _flush(tableId, start, end, wait);
+ }
+
+ @Override
+ public void compact(String tableName, Text start, Text end, boolean flush, boolean wait) throws AccumuloSecurityException, TableNotFoundException,
+ AccumuloException {
+ compact(tableName, start, end, new ArrayList<IteratorSetting>(), flush, wait);
+ }
+
+ @Override
+ public void compact(String tableName, Text start, Text end, List<IteratorSetting> iterators, boolean flush, boolean wait) throws AccumuloSecurityException,
+ TableNotFoundException, AccumuloException {
+ ArgumentChecker.notNull(tableName);
+ ByteBuffer EMPTY = ByteBuffer.allocate(0);
+
+ String tableId = Tables.getTableId(instance, tableName);
+
+ if (flush)
+ _flush(tableId, start, end, true);
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)), start == null ? EMPTY : TextUtil.getByteBuffer(start), end == null ? EMPTY
+ : TextUtil.getByteBuffer(end), ByteBuffer.wrap(IteratorUtil.encodeIteratorSettings(iterators)));
+
+ Map<String,String> opts = new HashMap<String,String>();
+ try {
+ doTableOperation(TableOperation.COMPACT, args, opts, wait);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void cancelCompaction(String tableName) throws AccumuloSecurityException, TableNotFoundException, AccumuloException {
+ String tableId = Tables.getTableId(instance, tableName);
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)));
+
+ Map<String,String> opts = new HashMap<String,String>();
+ try {
+ doTableOperation(TableOperation.COMPACT_CANCEL, args, opts, true);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ private void _flush(String tableId, Text start, Text end, boolean wait) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+
+ try {
+ long flushID;
+
+ // used to pass the table name. but the tableid associated with a table name could change between calls.
+ // so pass the tableid to both calls
+
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ flushID = client.initiateFlush(Tracer.traceInfo(), credentials, tableId);
+ break;
+ } catch (TTransportException tte) {
+ log.debug("Failed to call initiateFlush, retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ client.waitForFlush(Tracer.traceInfo(), credentials, tableId, TextUtil.getByteBuffer(start), TextUtil.getByteBuffer(end), flushID,
+ wait ? Long.MAX_VALUE : 1);
+ break;
+ } catch (TTransportException tte) {
+ log.debug("Failed to call initiateFlush, retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ } catch (ThriftSecurityException e) {
+ switch (e.getCode()) {
+ case TABLE_DOESNT_EXIST:
+ throw new TableNotFoundException(tableId, null, e.getMessage(), e);
+ default:
+ log.debug("flush security exception on table id " + tableId);
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ }
+ } catch (ThriftTableOperationException e) {
+ switch (e.getType()) {
+ case NOTFOUND:
+ throw new TableNotFoundException(e);
+ case OTHER:
+ default:
+ throw new AccumuloException(e.description, e);
+ }
+ } catch (Exception e) {
+ throw new AccumuloException(e);
+ }
+ }
+
+ /**
+ * Sets a property on a table
+ *
+ * @param tableName
+ * the name of the table
+ * @param property
+ * the name of a per-table property
+ * @param value
+ * the value to set a per-table property to
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ @Override
+ public void setProperty(final String tableName, final String property, final String value) throws AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, property, value);
+ MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() {
+ @Override
+ public void execute(MasterClientService.Client client) throws Exception {
+ client.setTableProperty(Tracer.traceInfo(), credentials, tableName, property, value);
+ }
+ });
+ }
+
+ /**
+ * Removes a property from a table
+ *
+ * @param tableName
+ * the name of the table
+ * @param property
+ * the name of a per-table property
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ @Override
+ public void removeProperty(final String tableName, final String property) throws AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, property);
+ MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() {
+ @Override
+ public void execute(MasterClientService.Client client) throws Exception {
+ client.removeTableProperty(Tracer.traceInfo(), credentials, tableName, property);
+ }
+ });
+ }
+
+ /**
+ * Gets properties of a table
+ *
+ * @param tableName
+ * the name of the table
+ * @return all properties visible by this table (system and per-table properties)
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public Iterable<Entry<String,String>> getProperties(final String tableName) throws AccumuloException, TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+ try {
+ return ServerClient.executeRaw(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() {
+ @Override
+ public Map<String,String> execute(ClientService.Client client) throws Exception {
+ return client.getTableConfiguration(Tracer.traceInfo(), credentials, tableName);
+ }
+ }).entrySet();
+ } catch (ThriftTableOperationException e) {
+ switch (e.getType()) {
+ case NOTFOUND:
+ throw new TableNotFoundException(e);
+ case OTHER:
+ default:
+ throw new AccumuloException(e.description, e);
+ }
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AccumuloException(e);
+ }
+
+ }
+
+ /**
+ * Sets a tables locality groups. A tables locality groups can be changed at any time.
+ *
+ * @param tableName
+ * the name of the table
+ * @param groups
+ * mapping of locality group names to column families in the locality group
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public void setLocalityGroups(String tableName, Map<String,Set<Text>> groups) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ // ensure locality groups do not overlap
+ HashSet<Text> all = new HashSet<Text>();
+ for (Entry<String,Set<Text>> entry : groups.entrySet()) {
+
+ if (!Collections.disjoint(all, entry.getValue())) {
+ throw new IllegalArgumentException("Group " + entry.getKey() + " overlaps with another group");
+ }
+
+ all.addAll(entry.getValue());
+ }
+
+ for (Entry<String,Set<Text>> entry : groups.entrySet()) {
+ Set<Text> colFams = entry.getValue();
+ String value = LocalityGroupUtil.encodeColumnFamilies(colFams);
+ setProperty(tableName, Property.TABLE_LOCALITY_GROUP_PREFIX + entry.getKey(), value);
+ }
+
+ setProperty(tableName, Property.TABLE_LOCALITY_GROUPS.getKey(), StringUtil.join(groups.keySet(), ","));
+
+ // remove anything extraneous
+ String prefix = Property.TABLE_LOCALITY_GROUP_PREFIX.getKey();
+ for (Entry<String,String> entry : getProperties(tableName)) {
+ String property = entry.getKey();
+ if (property.startsWith(prefix)) {
+ // this property configures a locality group, find out which
+ // one:
+ String[] parts = property.split("\\.");
+ String group = parts[parts.length - 1];
+
+ if (!groups.containsKey(group)) {
+ removeProperty(tableName, property);
+ }
+ }
+ }
+ }
+
+ /**
+ *
+ * Gets the locality groups currently set for a table.
+ *
+ * @param tableName
+ * the name of the table
+ * @return mapping of locality group names to column families in the locality group
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public Map<String,Set<Text>> getLocalityGroups(String tableName) throws AccumuloException, TableNotFoundException {
+ AccumuloConfiguration conf = new ConfigurationCopy(this.getProperties(tableName));
+ Map<String,Set<ByteSequence>> groups = LocalityGroupUtil.getLocalityGroups(conf);
+
+ Map<String,Set<Text>> groups2 = new HashMap<String,Set<Text>>();
+ for (Entry<String,Set<ByteSequence>> entry : groups.entrySet()) {
+
+ HashSet<Text> colFams = new HashSet<Text>();
+
+ for (ByteSequence bs : entry.getValue()) {
+ colFams.add(new Text(bs.toArray()));
+ }
+
+ groups2.put(entry.getKey(), colFams);
+ }
+
+ return groups2;
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param range
+ * a range to split
+ * @param maxSplits
+ * the maximum number of splits
+ * @return the range, split into smaller ranges that fall on boundaries of the table's split points as evenly as possible
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public Set<Range> splitRangeByTablets(String tableName, Range range, int maxSplits) throws AccumuloException, AccumuloSecurityException,
+ TableNotFoundException {
+ ArgumentChecker.notNull(tableName, range);
+ if (maxSplits < 1)
+ throw new IllegalArgumentException("maximum splits must be >= 1");
+ if (maxSplits == 1)
+ return Collections.singleton(range);
+
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+ String tableId = Tables.getTableId(instance, tableName);
+ TabletLocator tl = TabletLocator.getInstance(instance, new Text(tableId));
+ // its possible that the cache could contain complete, but old information about a tables tablets... so clear it
+ tl.invalidateCache();
+ while (!tl.binRanges(Collections.singletonList(range), binnedRanges, credentials).isEmpty()) {
+ if (!Tables.exists(instance, tableId))
+ throw new TableDeletedException(tableId);
+ if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
+ throw new TableOfflineException(instance, tableId);
+
+ log.warn("Unable to locate bins for specified range. Retrying.");
+ // sleep randomly between 100 and 200ms
+ UtilWaitThread.sleep(100 + (int) (Math.random() * 100));
+ binnedRanges.clear();
+ tl.invalidateCache();
+ }
+
+ // group key extents to get <= maxSplits
+ LinkedList<KeyExtent> unmergedExtents = new LinkedList<KeyExtent>();
+ List<KeyExtent> mergedExtents = new ArrayList<KeyExtent>();
+
+ for (Map<KeyExtent,List<Range>> map : binnedRanges.values())
+ unmergedExtents.addAll(map.keySet());
+
+ // the sort method is efficient for linked list
+ Collections.sort(unmergedExtents);
+
+ while (unmergedExtents.size() + mergedExtents.size() > maxSplits) {
+ if (unmergedExtents.size() >= 2) {
+ KeyExtent first = unmergedExtents.removeFirst();
+ KeyExtent second = unmergedExtents.removeFirst();
+ first.setEndRow(second.getEndRow());
+ mergedExtents.add(first);
+ } else {
+ mergedExtents.addAll(unmergedExtents);
+ unmergedExtents.clear();
+ unmergedExtents.addAll(mergedExtents);
+ mergedExtents.clear();
+ }
+
+ }
+
+ mergedExtents.addAll(unmergedExtents);
+
+ Set<Range> ranges = new HashSet<Range>();
+ for (KeyExtent k : mergedExtents)
+ ranges.add(k.toDataRange().clip(range));
+
+ return ranges;
+ }
+
+ @Override
+ public void importDirectory(String tableName, String dir, String failureDir, boolean setTime) throws IOException, AccumuloSecurityException,
+ TableNotFoundException, AccumuloException {
+ ArgumentChecker.notNull(tableName, dir, failureDir);
+ FileSystem fs = FileUtil.getFileSystem(CachedConfiguration.getInstance(), instance.getConfiguration());
+ Path dirPath = fs.makeQualified(new Path(dir));
+ Path failPath = fs.makeQualified(new Path(failureDir));
+ if (!fs.exists(dirPath))
+ throw new AccumuloException("Bulk import directory " + dir + " does not exist!");
+ if (!fs.exists(failPath))
+ throw new AccumuloException("Bulk import failure directory " + failureDir + " does not exist!");
+ FileStatus[] listStatus = fs.listStatus(failPath);
+ if (listStatus != null && listStatus.length != 0) {
+ if (listStatus.length == 1 && listStatus[0].isDir())
+ throw new AccumuloException("Bulk import directory " + failPath + " is a file");
+ throw new AccumuloException("Bulk import failure directory " + failPath + " is not empty");
+ }
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(dirPath.toString().getBytes(Constants.UTF8)),
+ ByteBuffer.wrap(failPath.toString().getBytes(Constants.UTF8)), ByteBuffer.wrap((setTime + "").getBytes(Constants.UTF8)));
+ Map<String,String> opts = new HashMap<String,String>();
+
+ try {
+ doTableOperation(TableOperation.BULK_IMPORT, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+ // return new BulkImportHelper(instance, credentials, tableName).importDirectory(new Path(dir), new Path(failureDir), numThreads, numAssignThreads,
+ // disableGC);
+ }
+
+ /**
+ *
+ * @param tableName
+ * the table to take offline
+ * @throws AccumuloException
+ * when there is a general accumulo error
+ * @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
- * @throws TableNotFoundException
+ */
+ @Override
+ public void offline(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
+
+ ArgumentChecker.notNull(tableName);
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)));
+ Map<String,String> opts = new HashMap<String,String>();
+
+ try {
+ doTableOperation(TableOperation.OFFLINE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ *
+ * @param tableName
+ * the table to take online
+ * @throws AccumuloException
+ * when there is a general accumulo error
+ * @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
- * @throws TableNotFoundException
+ */
+ @Override
+ public void online(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)));
+ Map<String,String> opts = new HashMap<String,String>();
+
+ try {
+ doTableOperation(TableOperation.ONLINE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Clears the tablet locator cache for a specified table
+ *
+ * @param tableName
+ * the name of the table
+ * @throws TableNotFoundException
+ * if table does not exist
+ */
+ @Override
+ public void clearLocatorCache(String tableName) throws TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+ TabletLocator tabLocator = TabletLocator.getInstance(instance, new Text(Tables.getTableId(instance, tableName)));
+ tabLocator.invalidateCache();
+ }
+
+ /**
+ * Get a mapping of table name to internal table id.
+ *
+ * @return the map from table name to internal table id
+ */
+ @Override
+ public Map<String,String> tableIdMap() {
+ return Tables.getNameToIdMap(instance);
+ }
+
+ @Override
+ public Text getMaxRow(String tableName, Authorizations auths, Text startRow, boolean startInclusive, Text endRow, boolean endInclusive)
+ throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, auths);
+ Scanner scanner = instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials)).createScanner(tableName, auths);
+ return FindMax.findMax(scanner, startRow, startInclusive, endRow, endInclusive);
+ }
+
+ public static Map<String,String> getExportedProps(FileSystem fs, Path path) throws IOException {
+ HashMap<String,String> props = new HashMap<String,String>();
+
+ ZipInputStream zis = new ZipInputStream(fs.open(path));
+ try {
+ ZipEntry zipEntry;
+ while ((zipEntry = zis.getNextEntry()) != null) {
+ if (zipEntry.getName().equals(Constants.EXPORT_TABLE_CONFIG_FILE)) {
+ BufferedReader in = new BufferedReader(new InputStreamReader(zis, Constants.UTF8));
+ String line;
+ while ((line = in.readLine()) != null) {
+ String sa[] = line.split("=", 2);
+ props.put(sa[0], sa[1]);
+ }
+
+ break;
+ }
+ }
+ } finally {
+ zis.close();
+ }
+ return props;
+ }
+
+ @Override
+ public void importTable(String tableName, String importDir) throws TableExistsException, AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, importDir);
+
+ try {
+ FileSystem fs = FileUtil.getFileSystem(CachedConfiguration.getInstance(), instance.getConfiguration());
+
+ Map<String,String> props = getExportedProps(fs, new Path(importDir, Constants.EXPORT_FILE));
+
+ for (Entry<String,String> prop : props.entrySet()) {
+ if (Property.isClassProperty(prop.getKey()) && !prop.getValue().contains(Constants.CORE_PACKAGE_NAME)) {
+ Logger.getLogger(this.getClass()).info(
+ "Imported table sets '" + prop.getKey() + "' to '" + prop.getValue() + "'. Ensure this class is on Accumulo classpath.");
+ }
+ }
+
+ } catch (IOException ioe) {
+ Logger.getLogger(this.getClass()).warn("Failed to check if imported table references external java classes : " + ioe.getMessage());
+ }
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(importDir.getBytes(Constants.UTF8)));
+
+ Map<String,String> opts = Collections.emptyMap();
+
+ try {
+ doTableOperation(TableOperation.IMPORT, args, opts);
+ } catch (TableNotFoundException e1) {
+ // should not happen
+ throw new RuntimeException(e1);
+ }
+
+ }
+
+ @Override
+ public void exportTable(String tableName, String exportDir) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, exportDir);
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(exportDir.getBytes(Constants.UTF8)));
+
+ Map<String,String> opts = Collections.emptyMap();
+
+ try {
+ doTableOperation(TableOperation.EXPORT, args, opts);
+ } catch (TableExistsException e1) {
+ // should not happen
+ throw new RuntimeException(e1);
+ }
+ }
+
+ @Override
+ public boolean testClassLoad(final String tableName, final String className, final String asTypeName) throws TableNotFoundException, AccumuloException,
+ AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, className, asTypeName);
+
+ try {
+ return ServerClient.executeRaw(instance, new ClientExecReturn<Boolean,ClientService.Client>() {
+ @Override
+ public Boolean execute(ClientService.Client client) throws Exception {
+ return client.checkTableClass(Tracer.traceInfo(), credentials, tableName, className, asTypeName);
+ }
+ });
+ } catch (ThriftTableOperationException e) {
+ switch (e.getType()) {
+ case NOTFOUND:
+ throw new TableNotFoundException(e);
+ case OTHER:
+ default:
+ throw new AccumuloException(e.description, e);
+ }
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AccumuloException(e);
+ }
+ }
+
+ @Override
+ public void attachIterator(String tableName, IteratorSetting setting, EnumSet<IteratorScope> scopes) throws AccumuloSecurityException, AccumuloException,
+ TableNotFoundException {
+ testClassLoad(tableName, setting.getIteratorClass(), SortedKeyValueIterator.class.getName());
+ super.attachIterator(tableName, setting, scopes);
+ }
+
+ @Override
+ public int addConstraint(String tableName, String constraintClassName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ testClassLoad(tableName, constraintClassName, Constraint.class.getName());
+ return super.addConstraint(tableName, constraintClassName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
index e00fcc8,0000000..1132e74
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
@@@ -1,414 -1,0 +1,408 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.impl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.RowIterator;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Column;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.KeyValue;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVIterator;
+import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
+import org.apache.accumulo.core.iterators.system.ColumnQualifierFilter;
+import org.apache.accumulo.core.iterators.system.DeletingIterator;
+import org.apache.accumulo.core.iterators.system.MultiIterator;
+import org.apache.accumulo.core.iterators.system.VisibilityFilter;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.LocalityGroupUtil;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.Text;
+
+class OfflineIterator implements Iterator<Entry<Key,Value>> {
+
+ static class OfflineIteratorEnvironment implements IteratorEnvironment {
+ @Override
+ public SortedKeyValueIterator<Key,Value> reserveMapFileReader(String mapFileName) throws IOException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public AccumuloConfiguration getConfig() {
+ return AccumuloConfiguration.getDefaultConfiguration();
+ }
+
+ @Override
+ public IteratorScope getIteratorScope() {
+ return IteratorScope.scan;
+ }
+
+ @Override
+ public boolean isFullMajorCompaction() {
+ return false;
+ }
+
+ private ArrayList<SortedKeyValueIterator<Key,Value>> topLevelIterators = new ArrayList<SortedKeyValueIterator<Key,Value>>();
+
+ @Override
+ public void registerSideChannel(SortedKeyValueIterator<Key,Value> iter) {
+ topLevelIterators.add(iter);
+ }
+
+ SortedKeyValueIterator<Key,Value> getTopLevelIterator(SortedKeyValueIterator<Key,Value> iter) {
+ if (topLevelIterators.isEmpty())
+ return iter;
+ ArrayList<SortedKeyValueIterator<Key,Value>> allIters = new ArrayList<SortedKeyValueIterator<Key,Value>>(topLevelIterators);
+ allIters.add(iter);
+ return new MultiIterator(allIters, false);
+ }
+ }
+
+ private SortedKeyValueIterator<Key,Value> iter;
+ private Range range;
+ private KeyExtent currentExtent;
+ private Connector conn;
+ private String tableId;
+ private Authorizations authorizations;
+ private Instance instance;
+ private ScannerOptions options;
+ private ArrayList<SortedKeyValueIterator<Key,Value>> readers;
+ private AccumuloConfiguration config;
+
- /**
- * @param instance
- * @param credentials
- * @param authorizations
- * @param table
- */
+ public OfflineIterator(ScannerOptions options, Instance instance, TCredentials credentials, Authorizations authorizations, Text table, Range range) {
+ this.options = new ScannerOptions(options);
+ this.instance = instance;
+ this.range = range;
+
+ if (this.options.fetchedColumns.size() > 0) {
+ this.range = range.bound(this.options.fetchedColumns.first(), this.options.fetchedColumns.last());
+ }
+
+ this.tableId = table.toString();
+ this.authorizations = authorizations;
+ this.readers = new ArrayList<SortedKeyValueIterator<Key,Value>>();
+
+ try {
+ conn = instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials));
+ config = new ConfigurationCopy(conn.instanceOperations().getSiteConfiguration());
+ nextTablet();
+
+ while (iter != null && !iter.hasTop())
+ nextTablet();
+
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return iter != null && iter.hasTop();
+ }
+
+ @Override
+ public Entry<Key,Value> next() {
+ try {
+ byte[] v = iter.getTopValue().get();
+ // copy just like tablet server does, do this before calling next
+ KeyValue ret = new KeyValue(new Key(iter.getTopKey()), Arrays.copyOf(v, v.length));
+
+ iter.next();
+
+ while (iter != null && !iter.hasTop())
+ nextTablet();
+
+ return ret;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * @throws TableNotFoundException
+ * @throws IOException
+ * @throws AccumuloException
+ *
+ */
+ private void nextTablet() throws TableNotFoundException, AccumuloException, IOException {
+
+ Range nextRange = null;
+
+ if (currentExtent == null) {
+ Text startRow;
+
+ if (range.getStartKey() != null)
+ startRow = range.getStartKey().getRow();
+ else
+ startRow = new Text();
+
+ nextRange = new Range(new KeyExtent(new Text(tableId), startRow, null).getMetadataEntry(), true, null, false);
+ } else {
+
+ if (currentExtent.getEndRow() == null) {
+ iter = null;
+ return;
+ }
+
+ if (range.afterEndKey(new Key(currentExtent.getEndRow()).followingKey(PartialKey.ROW))) {
+ iter = null;
+ return;
+ }
+
+ nextRange = new Range(currentExtent.getMetadataEntry(), false, null, false);
+ }
+
+ List<String> relFiles = new ArrayList<String>();
+
+ Pair<KeyExtent,String> eloc = getTabletFiles(nextRange, relFiles);
+
+ while (eloc.getSecond() != null) {
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ Tables.clearCache(instance);
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ throw new AccumuloException("Table is online " + tableId + " cannot scan tablet in offline mode " + eloc.getFirst());
+ }
+ }
+
+ UtilWaitThread.sleep(250);
+
+ eloc = getTabletFiles(nextRange, relFiles);
+ }
+
+ KeyExtent extent = eloc.getFirst();
+
+ if (!extent.getTableId().toString().equals(tableId)) {
+ throw new AccumuloException(" did not find tablets for table " + tableId + " " + extent);
+ }
+
+ if (currentExtent != null && !extent.isPreviousExtent(currentExtent))
+ throw new AccumuloException(" " + currentExtent + " is not previous extent " + extent);
+
+ String tablesDir = Constants.getTablesDir(config);
+ List<String> absFiles = new ArrayList<String>();
+ for (String relPath : relFiles) {
+ if (relPath.startsWith(".."))
+ absFiles.add(tablesDir + relPath.substring(2));
+ else
+ absFiles.add(tablesDir + "/" + tableId + relPath);
+ }
+
+ iter = createIterator(extent, absFiles);
+ iter.seek(range, LocalityGroupUtil.families(options.fetchedColumns), options.fetchedColumns.size() == 0 ? false : true);
+ currentExtent = extent;
+
+ }
+
+ private Pair<KeyExtent,String> getTabletFiles(Range nextRange, List<String> relFiles) throws TableNotFoundException {
+ Scanner scanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ scanner.setBatchSize(100);
+ scanner.setRange(nextRange);
+
+ RowIterator rowIter = new RowIterator(scanner);
+ Iterator<Entry<Key,Value>> row = rowIter.next();
+
+ KeyExtent extent = null;
+ String location = null;
+
+ while (row.hasNext()) {
+ Entry<Key,Value> entry = row.next();
+ Key key = entry.getKey();
+
+ if (key.getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
+ relFiles.add(key.getColumnQualifier().toString());
+ }
+
+ if (key.getColumnFamily().equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY)
+ || key.getColumnFamily().equals(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY)) {
+ location = entry.getValue().toString();
+ }
+
+ if (Constants.METADATA_PREV_ROW_COLUMN.hasColumns(key)) {
+ extent = new KeyExtent(key.getRow(), entry.getValue());
+ }
+
+ }
+ return new Pair<KeyExtent,String>(extent, location);
+ }
+
+ /**
+ * @param absFiles
+ * @return
+ * @throws AccumuloException
+ * @throws TableNotFoundException
+ * @throws IOException
+ */
+ private SortedKeyValueIterator<Key,Value> createIterator(KeyExtent extent, List<String> absFiles) throws TableNotFoundException, AccumuloException,
+ IOException {
+
+ // TODO share code w/ tablet - ACCUMULO-1303
+ AccumuloConfiguration acuTableConf = AccumuloConfiguration.getTableConfiguration(conn, tableId);
+
+ Configuration conf = CachedConfiguration.getInstance();
+
+ FileSystem fs = FileUtil.getFileSystem(conf, config);
+
+ for (SortedKeyValueIterator<Key,Value> reader : readers) {
+ ((FileSKVIterator) reader).close();
+ }
+
+ readers.clear();
+
+ // TODO need to close files - ACCUMULO-1303
+ for (String file : absFiles) {
+ FileSKVIterator reader = FileOperations.getInstance().openReader(file, false, fs, conf, acuTableConf, null, null);
+ readers.add(reader);
+ }
+
+ MultiIterator multiIter = new MultiIterator(readers, extent);
+
+ OfflineIteratorEnvironment iterEnv = new OfflineIteratorEnvironment();
+
+ DeletingIterator delIter = new DeletingIterator(multiIter, false);
+
+ ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(delIter);
+
+ ColumnQualifierFilter colFilter = new ColumnQualifierFilter(cfsi, new HashSet<Column>(options.fetchedColumns));
+
+ byte[] defaultSecurityLabel;
+
+ ColumnVisibility cv = new ColumnVisibility(acuTableConf.get(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY));
+ defaultSecurityLabel = cv.getExpression();
+
+ VisibilityFilter visFilter = new VisibilityFilter(colFilter, authorizations, defaultSecurityLabel);
+
+ return iterEnv.getTopLevelIterator(IteratorUtil.loadIterators(IteratorScope.scan, visFilter, extent, acuTableConf, options.serverSideIteratorList,
+ options.serverSideIteratorOptions, iterEnv, false));
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+}
+
+/**
+ *
+ */
+public class OfflineScanner extends ScannerOptions implements Scanner {
+
+ private int batchSize;
+ private int timeOut;
+ private Range range;
+
+ private Instance instance;
+ private TCredentials credentials;
+ private Authorizations authorizations;
+ private Text tableId;
+
+ public OfflineScanner(Instance instance, TCredentials credentials, String tableId, Authorizations authorizations) {
+ ArgumentChecker.notNull(instance, credentials, tableId, authorizations);
+ this.instance = instance;
+ this.credentials = credentials;
+ this.tableId = new Text(tableId);
+ this.range = new Range((Key) null, (Key) null);
+
+ this.authorizations = authorizations;
+
+ this.batchSize = Constants.SCAN_BATCH_SIZE;
+ this.timeOut = Integer.MAX_VALUE;
+ }
+
+ @Deprecated
+ @Override
+ public void setTimeOut(int timeOut) {
+ this.timeOut = timeOut;
+ }
+
+ @Deprecated
+ @Override
+ public int getTimeOut() {
+ return timeOut;
+ }
+
+ @Override
+ public void setRange(Range range) {
+ this.range = range;
+ }
+
+ @Override
+ public Range getRange() {
+ return range;
+ }
+
+ @Override
+ public void setBatchSize(int size) {
+ this.batchSize = size;
+ }
+
+ @Override
+ public int getBatchSize() {
+ return batchSize;
+ }
+
+ @Override
+ public void enableIsolation() {
+
+ }
+
+ @Override
+ public void disableIsolation() {
+
+ }
+
+ @Override
+ public Iterator<Entry<Key,Value>> iterator() {
+ return new OfflineIterator(this, instance, credentials, authorizations, tableId, range);
+ }
+
+}
[29/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
index cb54856,0000000..1a029dc
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
@@@ -1,273 -1,0 +1,272 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce.lib.util;
+
+import java.nio.charset.Charset;
+
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * @since 1.5.0
+ */
+public class ConfiguratorBase {
+
+ /**
+ * Configuration keys for {@link Instance#getConnector(String, AuthenticationToken)}.
+ *
+ * @since 1.5.0
+ */
+ public static enum ConnectorInfo {
+ IS_CONFIGURED, PRINCIPAL, TOKEN, TOKEN_CLASS
+ }
+
+ /**
+ * Configuration keys for {@link Instance}, {@link ZooKeeperInstance}, and {@link MockInstance}.
+ *
+ * @since 1.5.0
+ */
+ protected static enum InstanceOpts {
+ TYPE, NAME, ZOO_KEEPERS;
+ }
+
+ /**
+ * Configuration keys for general configuration options.
+ *
+ * @since 1.5.0
+ */
+ protected static enum GeneralOpts {
+ LOG_LEVEL
+ }
+
+ /**
+ * Provides a configuration key for a given feature enum, prefixed by the implementingClass
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param e
+ * the enum used to provide the unique part of the configuration key
+ * @return the configuration key
+ * @since 1.5.0
+ */
+ protected static String enumToConfKey(Class<?> implementingClass, Enum<?> e) {
+ return implementingClass.getSimpleName() + "." + e.getDeclaringClass().getSimpleName() + "." + StringUtils.camelize(e.name().toLowerCase());
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
+ * conversion to a string, and is not intended to be secure.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param principal
+ * a valid Accumulo user name
+ * @param token
+ * the user's password
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public static void setConnectorInfo(Class<?> implementingClass, Configuration conf, String principal, AuthenticationToken token)
+ throws AccumuloSecurityException {
+ if (isConnectorInfoSet(implementingClass, conf))
+ throw new IllegalStateException("Connector info for " + implementingClass.getSimpleName() + " can only be set once per job");
+
+ ArgumentChecker.notNull(principal, token);
+ conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL), principal);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN_CLASS), token.getClass().getCanonicalName());
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN), CredentialHelper.tokenAsBase64(token));
+ }
+
+ /**
+ * Determines if the connector info has already been set for this instance.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the connector info has already been set, false otherwise
+ * @since 1.5.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static Boolean isConnectorInfoSet(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), false);
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the principal
+ * @since 1.5.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static String getPrincipal(Class<?> implementingClass, Configuration conf) {
+ return conf.get(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL));
+ }
+
+ /**
+ * Gets the serialized token class from the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the principal
+ * @since 1.5.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static String getTokenClass(Class<?> implementingClass, Configuration conf) {
+ return conf.get(enumToConfKey(implementingClass, ConnectorInfo.TOKEN_CLASS));
+ }
+
+ /**
+ * Gets the password from the configuration. WARNING: The password is stored in the Configuration and shared with all MapReduce tasks; It is BASE64 encoded to
+ * provide a charset safe conversion to a string, and is not intended to be secure.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the decoded principal's authentication token
+ * @since 1.5.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static byte[] getToken(Class<?> implementingClass, Configuration conf) {
+ return Base64.decodeBase64(conf.get(enumToConfKey(implementingClass, ConnectorInfo.TOKEN), "").getBytes(Charset.forName("UTF-8")));
+ }
+
+ /**
+ * Configures a {@link ZooKeeperInstance} for this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param instanceName
+ * the Accumulo instance name
+ * @param zooKeepers
+ * a comma-separated list of zookeeper servers
+ * @since 1.5.0
+ */
+ public static void setZooKeeperInstance(Class<?> implementingClass, Configuration conf, String instanceName, String zooKeepers) {
+ String key = enumToConfKey(implementingClass, InstanceOpts.TYPE);
+ if (!conf.get(key, "").isEmpty())
+ throw new IllegalStateException("Instance info can only be set once per job; it has already been configured with " + conf.get(key));
+ conf.set(key, "ZooKeeperInstance");
+
+ ArgumentChecker.notNull(instanceName, zooKeepers);
+ conf.set(enumToConfKey(implementingClass, InstanceOpts.NAME), instanceName);
+ conf.set(enumToConfKey(implementingClass, InstanceOpts.ZOO_KEEPERS), zooKeepers);
+ }
+
+ /**
+ * Configures a {@link MockInstance} for this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param instanceName
+ * the Accumulo instance name
+ * @since 1.5.0
+ */
+ public static void setMockInstance(Class<?> implementingClass, Configuration conf, String instanceName) {
+ String key = enumToConfKey(implementingClass, InstanceOpts.TYPE);
+ if (!conf.get(key, "").isEmpty())
+ throw new IllegalStateException("Instance info can only be set once per job; it has already been configured with " + conf.get(key));
+ conf.set(key, "MockInstance");
+
+ ArgumentChecker.notNull(instanceName);
+ conf.set(enumToConfKey(implementingClass, InstanceOpts.NAME), instanceName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link Instance} based on the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return an Accumulo instance
+ * @since 1.5.0
+ * @see #setZooKeeperInstance(Class, Configuration, String, String)
+ * @see #setMockInstance(Class, Configuration, String)
+ */
+ public static Instance getInstance(Class<?> implementingClass, Configuration conf) {
+ String instanceType = conf.get(enumToConfKey(implementingClass, InstanceOpts.TYPE), "");
+ if ("MockInstance".equals(instanceType))
+ return new MockInstance(conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME)));
+ else if ("ZooKeeperInstance".equals(instanceType)) {
+ return new ZooKeeperInstance(conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME)), conf.get(enumToConfKey(implementingClass,
+ InstanceOpts.ZOO_KEEPERS)));
+ } else if (instanceType.isEmpty())
+ throw new IllegalStateException("Instance has not been configured for " + implementingClass.getSimpleName());
+ else
+ throw new IllegalStateException("Unrecognized instance type " + instanceType);
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param level
+ * the logging level
+ * @since 1.5.0
+ */
+ public static void setLogLevel(Class<?> implementingClass, Configuration conf, Level level) {
+ ArgumentChecker.notNull(level);
+ Logger.getLogger(implementingClass).setLevel(level);
+ conf.setInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), level.toInt());
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the log level
+ * @since 1.5.0
+ * @see #setLogLevel(Class, Configuration, Level)
+ */
+ public static Level getLogLevel(Class<?> implementingClass, Configuration conf) {
+ return Level.toLevel(conf.getInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), Level.INFO.toInt()));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/mock/MockBatchDeleter.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mock/MockBatchDeleter.java
index 67362a2,0000000..6f321ff
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockBatchDeleter.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockBatchDeleter.java
@@@ -1,77 -1,0 +1,73 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mock;
+
+import java.util.Iterator;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.BatchDeleter;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
+
+/**
+ * {@link BatchDeleter} for a {@link MockAccumulo} instance. Behaves similarly to a regular {@link BatchDeleter}, with a few exceptions:
+ * <ol>
+ * <li>There is no waiting for memory to fill before flushing</li>
+ * <li>Only one thread is used for writing</li>
+ * </ol>
+ *
+ * Otherwise, it behaves as expected.
+ */
+public class MockBatchDeleter extends MockBatchScanner implements BatchDeleter {
+
+ private final MockAccumulo acc;
+ private final String tableName;
+
+ /**
+ * Create a {@link BatchDeleter} for the specified instance on the specified table where the writer uses the specified {@link Authorizations}.
- *
- * @param acc
- * @param tableName
- * @param auths
+ */
+ public MockBatchDeleter(MockAccumulo acc, String tableName, Authorizations auths) {
+ super(acc.tables.get(tableName), auths);
+ this.acc = acc;
+ this.tableName = tableName;
+ }
+
+ @Override
+ public void delete() throws MutationsRejectedException, TableNotFoundException {
+
+ BatchWriter writer = new MockBatchWriter(acc, tableName);
+ try {
+ Iterator<Entry<Key,Value>> iter = super.iterator();
+ while (iter.hasNext()) {
+ Entry<Key,Value> next = iter.next();
+ Key k = next.getKey();
+ Mutation m = new Mutation(k.getRow());
+ m.putDelete(k.getColumnFamily(), k.getColumnQualifier(), new ColumnVisibility(k.getColumnVisibility()), k.getTimestamp());
+ writer.addMutation(m);
+ }
+ } finally {
+ writer.close();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java
index 34eb3de,0000000..cb9481f
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java
@@@ -1,133 -1,0 +1,90 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mock;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.admin.ActiveCompaction;
+import org.apache.accumulo.core.client.admin.ActiveScan;
+import org.apache.accumulo.core.client.admin.InstanceOperations;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+
+/**
+ *
+ */
+public class MockInstanceOperations implements InstanceOperations {
+ MockAccumulo acu;
+
- /**
- * @param acu
- */
+ public MockInstanceOperations(MockAccumulo acu) {
+ this.acu = acu;
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#setProperty(java.lang.String, java.lang.String)
- */
+ @Override
+ public void setProperty(String property, String value) throws AccumuloException, AccumuloSecurityException {
+ acu.setProperty(property, value);
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#removeProperty(java.lang.String)
- */
+ @Override
+ public void removeProperty(String property) throws AccumuloException, AccumuloSecurityException {
+ acu.removeProperty(property);
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getSystemConfiguration()
- */
+ @Override
+ public Map<String,String> getSystemConfiguration() throws AccumuloException, AccumuloSecurityException {
+ return acu.systemProperties;
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getSiteConfiguration()
- */
+ @Override
+ public Map<String,String> getSiteConfiguration() throws AccumuloException, AccumuloSecurityException {
+ return acu.systemProperties;
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getTabletServers()
- */
+ @Override
+ public List<String> getTabletServers() {
+ return new ArrayList<String>();
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getActiveScans(java.lang.String)
- */
+ @Override
+ public List<ActiveScan> getActiveScans(String tserver) throws AccumuloException, AccumuloSecurityException {
+ return new ArrayList<ActiveScan>();
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#testClassLoad(java.lang.String, java.lang.String)
- */
+ @Override
+ public boolean testClassLoad(String className, String asTypeName) throws AccumuloException, AccumuloSecurityException {
+ try {
+ AccumuloVFSClassLoader.loadClass(className, Class.forName(asTypeName));
+ } catch (ClassNotFoundException e) {
+ e.printStackTrace();
+ return false;
+ }
+ return true;
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getActiveCompactions(java.lang.String)
- */
+ @Override
+ public List<ActiveCompaction> getActiveCompactions(String tserver) throws AccumuloException, AccumuloSecurityException {
+ return new ArrayList<ActiveCompaction>();
+ }
+
+ @Override
+ public void ping(String tserver) throws AccumuloException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/data/ColumnUpdate.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/ColumnUpdate.java
index bfba00f,0000000..78f2d15
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/data/ColumnUpdate.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/ColumnUpdate.java
@@@ -1,110 -1,0 +1,109 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
+import java.util.Arrays;
+
+/**
+ * A single column and value pair within a mutation
+ *
+ */
+
+public class ColumnUpdate {
+
+ private byte[] columnFamily;
+ private byte[] columnQualifier;
+ private byte[] columnVisibility;
+ private long timestamp;
+ private boolean hasTimestamp;
+ private byte[] val;
+ private boolean deleted;
+
+ public ColumnUpdate(byte[] cf, byte[] cq, byte[] cv, boolean hasts, long ts, boolean deleted, byte[] val) {
+ this.columnFamily = cf;
+ this.columnQualifier = cq;
+ this.columnVisibility = cv;
+ this.hasTimestamp = hasts;
+ this.timestamp = ts;
+ this.deleted = deleted;
+ this.val = val;
+ }
+
+ /**
+ * @deprecated use setTimestamp(long);
- * @param timestamp
+ */
+ @Deprecated
+ public void setSystemTimestamp(long timestamp) {
+ if (hasTimestamp)
+ throw new IllegalStateException("Cannot set system timestamp when user set a timestamp");
+ }
+
+ public boolean hasTimestamp() {
+ return hasTimestamp;
+ }
+
+ /**
+ * Returns the column
+ *
+ */
+ public byte[] getColumnFamily() {
+ return columnFamily;
+ }
+
+ public byte[] getColumnQualifier() {
+ return columnQualifier;
+ }
+
+ public byte[] getColumnVisibility() {
+ return columnVisibility;
+ }
+
+ public long getTimestamp() {
+ return this.timestamp;
+ }
+
+ public boolean isDeleted() {
+ return this.deleted;
+ }
+
+ public byte[] getValue() {
+ return this.val;
+ }
+
+ @Override
+ public String toString() {
+ return Arrays.toString(columnFamily) + ":" + Arrays.toString(columnQualifier) + " ["
+ + Arrays.toString(columnVisibility) + "] " + (hasTimestamp ? timestamp : "NO_TIME_STAMP") + " " + Arrays.toString(val) + " " + deleted;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof ColumnUpdate))
+ return false;
+ ColumnUpdate upd = (ColumnUpdate) obj;
+ return Arrays.equals(getColumnFamily(), upd.getColumnFamily()) && Arrays.equals(getColumnQualifier(), upd.getColumnQualifier())
+ && Arrays.equals(getColumnVisibility(), upd.getColumnVisibility()) && isDeleted() == upd.isDeleted() && Arrays.equals(getValue(), upd.getValue())
+ && hasTimestamp() == upd.hasTimestamp() && getTimestamp() == upd.getTimestamp();
+ }
+
+ @Override
+ public int hashCode() {
+ return Arrays.hashCode(columnFamily) + Arrays.hashCode(columnQualifier) + Arrays.hashCode(columnVisibility)
+ + (hasTimestamp ? (Boolean.TRUE.hashCode() + Long.valueOf(timestamp).hashCode()) : Boolean.FALSE.hashCode())
+ + (deleted ? Boolean.TRUE.hashCode() : (Boolean.FALSE.hashCode() + Arrays.hashCode(val)));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/data/Key.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Key.java
index 4b6867f,0000000..2b44359
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/data/Key.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Key.java
@@@ -1,863 -1,0 +1,864 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
+/**
+ * This is the Key used to store and access individual values in Accumulo. A Key is a tuple composed of a row, column family, column qualifier,
+ * column visibility, timestamp, and delete marker.
+ *
+ * Keys are comparable and therefore have a sorted order defined by {@link #compareTo(Key)}.
+ *
+ */
+
+import static org.apache.accumulo.core.util.ByteBufferUtil.toBytes;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.thrift.TKey;
+import org.apache.accumulo.core.data.thrift.TKeyValue;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.io.WritableUtils;
+
+public class Key implements WritableComparable<Key>, Cloneable {
+
+ protected byte[] row;
+ protected byte[] colFamily;
+ protected byte[] colQualifier;
+ protected byte[] colVisibility;
+ protected long timestamp;
+ protected boolean deleted;
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof Key)
+ return this.equals((Key) o, PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME_DEL);
+ return false;
+ }
+
+ private static final byte EMPTY_BYTES[] = new byte[0];
+
+ private byte[] copyIfNeeded(byte ba[], int off, int len, boolean copyData) {
+ if (len == 0)
+ return EMPTY_BYTES;
+
+ if (!copyData && ba.length == len && off == 0)
+ return ba;
+
+ byte[] copy = new byte[len];
+ System.arraycopy(ba, off, copy, 0, len);
+ return copy;
+ }
+
+ private final void init(byte r[], int rOff, int rLen, byte cf[], int cfOff, int cfLen, byte cq[], int cqOff, int cqLen, byte cv[], int cvOff, int cvLen,
+ long ts, boolean del, boolean copy) {
+ row = copyIfNeeded(r, rOff, rLen, copy);
+ colFamily = copyIfNeeded(cf, cfOff, cfLen, copy);
+ colQualifier = copyIfNeeded(cq, cqOff, cqLen, copy);
+ colVisibility = copyIfNeeded(cv, cvOff, cvLen, copy);
+ timestamp = ts;
+ deleted = del;
+ }
+
+ /**
+ * Creates a key with empty row, empty column family, empty column qualifier, empty column visibility, timestamp {@link Long#MAX_VALUE}, and delete marker
+ * false.
+ */
+ public Key() {
+ row = EMPTY_BYTES;
+ colFamily = EMPTY_BYTES;
+ colQualifier = EMPTY_BYTES;
+ colVisibility = EMPTY_BYTES;
+ timestamp = Long.MAX_VALUE;
+ deleted = false;
+ }
+
+ /**
+ * Creates a key with the specified row, empty column family, empty column qualifier, empty column visibility, timestamp {@link Long#MAX_VALUE}, and delete
+ * marker false.
+ */
+ public Key(Text row) {
+ init(row.getBytes(), 0, row.getLength(), EMPTY_BYTES, 0, 0, EMPTY_BYTES, 0, 0, EMPTY_BYTES, 0, 0, Long.MAX_VALUE, false, true);
+ }
+
+ /**
+ * Creates a key with the specified row, empty column family, empty column qualifier, empty column visibility, the specified timestamp, and delete marker
+ * false.
+ */
+ public Key(Text row, long ts) {
+ this(row);
+ timestamp = ts;
+ }
+
+ public Key(byte row[], int rOff, int rLen, byte cf[], int cfOff, int cfLen, byte cq[], int cqOff, int cqLen, byte cv[], int cvOff, int cvLen, long ts) {
+ init(row, rOff, rLen, cf, cfOff, cfLen, cq, cqOff, cqLen, cv, cvOff, cvLen, ts, false, true);
+ }
+
+ public Key(byte[] row, byte[] colFamily, byte[] colQualifier, byte[] colVisibility, long timestamp) {
+ this(row, colFamily, colQualifier, colVisibility, timestamp, false, true);
+ }
+
+ public Key(byte[] row, byte[] cf, byte[] cq, byte[] cv, long ts, boolean deleted) {
+ this(row, cf, cq, cv, ts, deleted, true);
+ }
+
+ public Key(byte[] row, byte[] cf, byte[] cq, byte[] cv, long ts, boolean deleted, boolean copy) {
+ init(row, 0, row.length, cf, 0, cf.length, cq, 0, cq.length, cv, 0, cv.length, ts, deleted, copy);
+ }
+
+ /**
+ * Creates a key with the specified row, the specified column family, empty column qualifier, empty column visibility, timestamp {@link Long#MAX_VALUE}, and
+ * delete marker false.
+ */
+ public Key(Text row, Text cf) {
+ init(row.getBytes(), 0, row.getLength(), cf.getBytes(), 0, cf.getLength(), EMPTY_BYTES, 0, 0, EMPTY_BYTES, 0, 0, Long.MAX_VALUE, false, true);
+ }
+
+ /**
+ * Creates a key with the specified row, the specified column family, the specified column qualifier, empty column visibility, timestamp
+ * {@link Long#MAX_VALUE}, and delete marker false.
+ */
+ public Key(Text row, Text cf, Text cq) {
+ init(row.getBytes(), 0, row.getLength(), cf.getBytes(), 0, cf.getLength(), cq.getBytes(), 0, cq.getLength(), EMPTY_BYTES, 0, 0, Long.MAX_VALUE, false, true);
+ }
+
+ /**
+ * Creates a key with the specified row, the specified column family, the specified column qualifier, the specified column visibility, timestamp
+ * {@link Long#MAX_VALUE}, and delete marker false.
+ */
+ public Key(Text row, Text cf, Text cq, Text cv) {
+ init(row.getBytes(), 0, row.getLength(), cf.getBytes(), 0, cf.getLength(), cq.getBytes(), 0, cq.getLength(), cv.getBytes(), 0, cv.getLength(),
+ Long.MAX_VALUE, false, true);
+ }
+
+ /**
+ * Creates a key with the specified row, the specified column family, the specified column qualifier, empty column visibility, the specified timestamp, and
+ * delete marker false.
+ */
+ public Key(Text row, Text cf, Text cq, long ts) {
+ init(row.getBytes(), 0, row.getLength(), cf.getBytes(), 0, cf.getLength(), cq.getBytes(), 0, cq.getLength(), EMPTY_BYTES, 0, 0, ts, false, true);
+ }
+
+ /**
+ * Creates a key with the specified row, the specified column family, the specified column qualifier, the specified column visibility, the specified
+ * timestamp, and delete marker false.
+ */
+ public Key(Text row, Text cf, Text cq, Text cv, long ts) {
+ init(row.getBytes(), 0, row.getLength(), cf.getBytes(), 0, cf.getLength(), cq.getBytes(), 0, cq.getLength(), cv.getBytes(), 0, cv.getLength(), ts, false,
+ true);
+ }
+
+ /**
+ * Creates a key with the specified row, the specified column family, the specified column qualifier, the specified column visibility, the specified
+ * timestamp, and delete marker false.
+ */
+ public Key(Text row, Text cf, Text cq, ColumnVisibility cv, long ts) {
+ byte[] expr = cv.getExpression();
+ init(row.getBytes(), 0, row.getLength(), cf.getBytes(), 0, cf.getLength(), cq.getBytes(), 0, cq.getLength(), expr, 0, expr.length, ts, false, true);
+ }
+
+ /**
+ * Converts CharSequence to Text and creates a Key using {@link #Key(Text)}.
+ */
+ public Key(CharSequence row) {
+ this(new Text(row.toString()));
+ }
+
+ /**
+ * Converts CharSequence to Text and creates a Key using {@link #Key(Text,Text)}.
+ */
+ public Key(CharSequence row, CharSequence cf) {
+ this(new Text(row.toString()), new Text(cf.toString()));
+ }
+
+ /**
+ * Converts CharSequence to Text and creates a Key using {@link #Key(Text,Text,Text)}.
+ */
+ public Key(CharSequence row, CharSequence cf, CharSequence cq) {
+ this(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()));
+ }
+
+ /**
+ * Converts CharSequence to Text and creates a Key using {@link #Key(Text,Text,Text,Text)}.
+ */
+ public Key(CharSequence row, CharSequence cf, CharSequence cq, CharSequence cv) {
+ this(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()), new Text(cv.toString()));
+ }
+
+ /**
+ * Converts CharSequence to Text and creates a Key using {@link #Key(Text,Text,Text,long)}.
+ */
+ public Key(CharSequence row, CharSequence cf, CharSequence cq, long ts) {
+ this(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()), ts);
+ }
+
+ /**
+ * Converts CharSequence to Text and creates a Key using {@link #Key(Text,Text,Text,Text,long)}.
+ */
+ public Key(CharSequence row, CharSequence cf, CharSequence cq, CharSequence cv, long ts) {
+ this(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()), new Text(cv.toString()), ts);
+ }
+
+ /**
+ * Converts CharSequence to Text and creates a Key using {@link #Key(Text,Text,Text,ColumnVisibility,long)}.
+ */
+ public Key(CharSequence row, CharSequence cf, CharSequence cq, ColumnVisibility cv, long ts) {
+ this(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()), new Text(cv.getExpression()), ts);
+ }
+
+ private byte[] followingArray(byte ba[]) {
+ byte[] fba = new byte[ba.length + 1];
+ System.arraycopy(ba, 0, fba, 0, ba.length);
+ fba[ba.length] = (byte) 0x00;
+ return fba;
+ }
+
+ /**
+ * Returns a key that will sort immediately after this key.
+ *
+ * @param part
+ * PartialKey except {@link PartialKey#ROW_COLFAM_COLQUAL_COLVIS_TIME_DEL}
+ */
+ public Key followingKey(PartialKey part) {
+ Key returnKey = new Key();
+ switch (part) {
+ case ROW:
+ returnKey.row = followingArray(row);
+ break;
+ case ROW_COLFAM:
+ returnKey.row = row;
+ returnKey.colFamily = followingArray(colFamily);
+ break;
+ case ROW_COLFAM_COLQUAL:
+ returnKey.row = row;
+ returnKey.colFamily = colFamily;
+ returnKey.colQualifier = followingArray(colQualifier);
+ break;
+ case ROW_COLFAM_COLQUAL_COLVIS:
+ // This isn't useful for inserting into accumulo, but may be useful for lookups.
+ returnKey.row = row;
+ returnKey.colFamily = colFamily;
+ returnKey.colQualifier = colQualifier;
+ returnKey.colVisibility = followingArray(colVisibility);
+ break;
+ case ROW_COLFAM_COLQUAL_COLVIS_TIME:
+ returnKey.row = row;
+ returnKey.colFamily = colFamily;
+ returnKey.colQualifier = colQualifier;
+ returnKey.colVisibility = colVisibility;
+ returnKey.setTimestamp(timestamp - 1);
+ returnKey.deleted = false;
+ break;
+ default:
+ throw new IllegalArgumentException("Partial key specification " + part + " disallowed");
+ }
+ return returnKey;
+ }
+
+ /**
+ * Creates a key with the same row, column family, column qualifier, column visibility, timestamp, and delete marker as the given key.
+ */
+ public Key(Key other) {
+ set(other);
+ }
+
+ public Key(TKey tkey) {
+ this.row = toBytes(tkey.row);
+ this.colFamily = toBytes(tkey.colFamily);
+ this.colQualifier = toBytes(tkey.colQualifier);
+ this.colVisibility = toBytes(tkey.colVisibility);
+ this.timestamp = tkey.timestamp;
+ this.deleted = false;
+
+ if (row == null) {
+ throw new IllegalArgumentException("null row");
+ }
+ if (colFamily == null) {
+ throw new IllegalArgumentException("null column family");
+ }
+ if (colQualifier == null) {
+ throw new IllegalArgumentException("null column qualifier");
+ }
+ if (colVisibility == null) {
+ throw new IllegalArgumentException("null column visibility");
+ }
+ }
+
+ /**
+ * This method gives users control over allocation of Text objects by copying into the passed in text.
+ *
+ * @param r
+ * the key's row will be copied into this Text
+ * @return the Text that was passed in
+ */
+
+ public Text getRow(Text r) {
+ r.set(row, 0, row.length);
+ return r;
+ }
+
+ /**
+ * This method returns a pointer to the keys internal data and does not copy it.
+ *
+ * @return ByteSequence that points to the internal key row data.
+ */
+
+ public ByteSequence getRowData() {
+ return new ArrayByteSequence(row);
+ }
+
+ /**
+ * This method allocates a Text object and copies into it.
+ *
+ * @return Text containing the row field
+ */
+
+ public Text getRow() {
+ return getRow(new Text());
+ }
+
+ /**
+ * Efficiently compare the the row of a key w/o allocating a text object and copying the row into it.
+ *
+ * @param r
+ * row to compare to keys row
+ * @return same as {@link #getRow()}.compareTo(r)
+ */
+
+ public int compareRow(Text r) {
+ return WritableComparator.compareBytes(row, 0, row.length, r.getBytes(), 0, r.getLength());
+ }
+
+ /**
+ * This method returns a pointer to the keys internal data and does not copy it.
+ *
+ * @return ByteSequence that points to the internal key column family data.
+ */
+
+ public ByteSequence getColumnFamilyData() {
+ return new ArrayByteSequence(colFamily);
+ }
+
+ /**
+ * This method gives users control over allocation of Text objects by copying into the passed in text.
+ *
+ * @param cf
+ * the key's column family will be copied into this Text
+ * @return the Text that was passed in
+ */
+
+ public Text getColumnFamily(Text cf) {
+ cf.set(colFamily, 0, colFamily.length);
+ return cf;
+ }
+
+ /**
+ * This method allocates a Text object and copies into it.
+ *
+ * @return Text containing the column family field
+ */
+
+ public Text getColumnFamily() {
+ return getColumnFamily(new Text());
+ }
+
+ /**
+ * Efficiently compare the the column family of a key w/o allocating a text object and copying the column qualifier into it.
+ *
+ * @param cf
+ * column family to compare to keys column family
+ * @return same as {@link #getColumnFamily()}.compareTo(cf)
+ */
+
+ public int compareColumnFamily(Text cf) {
+ return WritableComparator.compareBytes(colFamily, 0, colFamily.length, cf.getBytes(), 0, cf.getLength());
+ }
+
+ /**
+ * This method returns a pointer to the keys internal data and does not copy it.
+ *
+ * @return ByteSequence that points to the internal key column qualifier data.
+ */
+
+ public ByteSequence getColumnQualifierData() {
+ return new ArrayByteSequence(colQualifier);
+ }
+
+ /**
+ * This method gives users control over allocation of Text objects by copying into the passed in text.
+ *
+ * @param cq
+ * the key's column qualifier will be copied into this Text
+ * @return the Text that was passed in
+ */
+
+ public Text getColumnQualifier(Text cq) {
+ cq.set(colQualifier, 0, colQualifier.length);
+ return cq;
+ }
+
+ /**
+ * This method allocates a Text object and copies into it.
+ *
+ * @return Text containing the column qualifier field
+ */
+
+ public Text getColumnQualifier() {
+ return getColumnQualifier(new Text());
+ }
+
+ /**
+ * Efficiently compare the the column qualifier of a key w/o allocating a text object and copying the column qualifier into it.
+ *
+ * @param cq
+ * column family to compare to keys column qualifier
+ * @return same as {@link #getColumnQualifier()}.compareTo(cq)
+ */
+
+ public int compareColumnQualifier(Text cq) {
+ return WritableComparator.compareBytes(colQualifier, 0, colQualifier.length, cq.getBytes(), 0, cq.getLength());
+ }
+
+ public void setTimestamp(long ts) {
+ this.timestamp = ts;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public boolean isDeleted() {
+ return deleted;
+ }
+
+ public void setDeleted(boolean deleted) {
+ this.deleted = deleted;
+ }
+
+ /**
+ * This method returns a pointer to the keys internal data and does not copy it.
+ *
+ * @return ByteSequence that points to the internal key column visibility data.
+ */
+
+ public ByteSequence getColumnVisibilityData() {
+ return new ArrayByteSequence(colVisibility);
+ }
+
+ /**
+ * This method allocates a Text object and copies into it.
+ *
+ * @return Text containing the column visibility field
+ */
+
+ public final Text getColumnVisibility() {
+ return getColumnVisibility(new Text());
+ }
+
+ /**
+ * This method gives users control over allocation of Text objects by copying into the passed in text.
+ *
+ * @param cv
+ * the key's column visibility will be copied into this Text
+ * @return the Text that was passed in
+ */
+
+ public final Text getColumnVisibility(Text cv) {
+ cv.set(colVisibility, 0, colVisibility.length);
+ return cv;
+ }
+
+ /**
+ * This method creates a new ColumnVisibility representing the column visibility for this key
+ *
+ * WARNING: using this method may inhibit performance since a new ColumnVisibility object is created on every call.
+ *
+ * @return A new object representing the column visibility field
+ * @since 1.5.0
+ */
+ public final ColumnVisibility getColumnVisibilityParsed() {
+ return new ColumnVisibility(colVisibility);
+ }
+
+ /**
+ * Sets this key's row, column family, column qualifier, column visibility, timestamp, and delete marker to be the same as another key's.
+ */
+ public void set(Key k) {
+ row = k.row;
+ colFamily = k.colFamily;
+ colQualifier = k.colQualifier;
+ colVisibility = k.colVisibility;
+ timestamp = k.timestamp;
+ deleted = k.deleted;
+
+ }
+
++ @Override
+ public void readFields(DataInput in) throws IOException {
+ // this method is a little screwy so it will be compatible with older
+ // code that serialized data
+
+ int colFamilyOffset = WritableUtils.readVInt(in);
+ int colQualifierOffset = WritableUtils.readVInt(in);
+ int colVisibilityOffset = WritableUtils.readVInt(in);
+ int totalLen = WritableUtils.readVInt(in);
+
+ row = new byte[colFamilyOffset];
+ colFamily = new byte[colQualifierOffset - colFamilyOffset];
+ colQualifier = new byte[colVisibilityOffset - colQualifierOffset];
+ colVisibility = new byte[totalLen - colVisibilityOffset];
+
+ in.readFully(row);
+ in.readFully(colFamily);
+ in.readFully(colQualifier);
+ in.readFully(colVisibility);
+
+ timestamp = WritableUtils.readVLong(in);
+ deleted = in.readBoolean();
+ }
+
++ @Override
+ public void write(DataOutput out) throws IOException {
+
+ int colFamilyOffset = row.length;
+ int colQualifierOffset = colFamilyOffset + colFamily.length;
+ int colVisibilityOffset = colQualifierOffset + colQualifier.length;
+ int totalLen = colVisibilityOffset + colVisibility.length;
+
+ WritableUtils.writeVInt(out, colFamilyOffset);
+ WritableUtils.writeVInt(out, colQualifierOffset);
+ WritableUtils.writeVInt(out, colVisibilityOffset);
+
+ WritableUtils.writeVInt(out, totalLen);
+
+ out.write(row);
+ out.write(colFamily);
+ out.write(colQualifier);
+ out.write(colVisibility);
+
+ WritableUtils.writeVLong(out, timestamp);
+ out.writeBoolean(deleted);
+ }
+
+ /**
+ * Compare part of a key. For example compare just the row and column family, and if those are equal then return true.
+ *
+ */
+
+ public boolean equals(Key other, PartialKey part) {
+ switch (part) {
+ case ROW:
+ return isEqual(row, other.row);
+ case ROW_COLFAM:
+ return isEqual(row, other.row) && isEqual(colFamily, other.colFamily);
+ case ROW_COLFAM_COLQUAL:
+ return isEqual(row, other.row) && isEqual(colFamily, other.colFamily) && isEqual(colQualifier, other.colQualifier);
+ case ROW_COLFAM_COLQUAL_COLVIS:
+ return isEqual(row, other.row) && isEqual(colFamily, other.colFamily) && isEqual(colQualifier, other.colQualifier)
+ && isEqual(colVisibility, other.colVisibility);
+ case ROW_COLFAM_COLQUAL_COLVIS_TIME:
+ return isEqual(row, other.row) && isEqual(colFamily, other.colFamily) && isEqual(colQualifier, other.colQualifier)
+ && isEqual(colVisibility, other.colVisibility) && timestamp == other.timestamp;
+ case ROW_COLFAM_COLQUAL_COLVIS_TIME_DEL:
+ return isEqual(row, other.row) && isEqual(colFamily, other.colFamily) && isEqual(colQualifier, other.colQualifier)
+ && isEqual(colVisibility, other.colVisibility) && timestamp == other.timestamp && deleted == other.deleted;
+ default:
+ throw new IllegalArgumentException("Unrecognized partial key specification " + part);
+ }
+ }
+
+ /**
+ * Compare elements of a key given by a {@link PartialKey}. For example, for {@link PartialKey#ROW_COLFAM}, compare just the row and column family. If the
+ * rows are not equal, return the result of the row comparison; otherwise, return the result of the column family comparison.
+ *
+ * @see #compareTo(Key)
+ */
+
+ public int compareTo(Key other, PartialKey part) {
+ // check for matching row
+ int result = WritableComparator.compareBytes(row, 0, row.length, other.row, 0, other.row.length);
+ if (result != 0 || part.equals(PartialKey.ROW))
+ return result;
+
+ // check for matching column family
+ result = WritableComparator.compareBytes(colFamily, 0, colFamily.length, other.colFamily, 0, other.colFamily.length);
+ if (result != 0 || part.equals(PartialKey.ROW_COLFAM))
+ return result;
+
+ // check for matching column qualifier
+ result = WritableComparator.compareBytes(colQualifier, 0, colQualifier.length, other.colQualifier, 0, other.colQualifier.length);
+ if (result != 0 || part.equals(PartialKey.ROW_COLFAM_COLQUAL))
+ return result;
+
+ // check for matching column visibility
+ result = WritableComparator.compareBytes(colVisibility, 0, colVisibility.length, other.colVisibility, 0, other.colVisibility.length);
+ if (result != 0 || part.equals(PartialKey.ROW_COLFAM_COLQUAL_COLVIS))
+ return result;
+
+ // check for matching timestamp
+ if (timestamp < other.timestamp)
+ result = 1;
+ else if (timestamp > other.timestamp)
+ result = -1;
+ else
+ result = 0;
+
+ if (result != 0 || part.equals(PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME))
+ return result;
+
+ // check for matching deleted flag
+ if (deleted)
+ result = other.deleted ? 0 : -1;
+ else
+ result = other.deleted ? 1 : 0;
+
+ return result;
+ }
+
+ /**
+ * Compare all elements of a key. The elements (row, column family, column qualifier, column visibility, timestamp, and delete marker) are compared in order
+ * until an unequal element is found. If the row is equal, then compare the column family, etc. The row, column family, column qualifier, and column
+ * visibility are compared lexographically and sorted ascending. The timestamps are compared numerically and sorted descending so that the most recent data
+ * comes first. Lastly, a delete marker of true sorts before a delete marker of false.
+ */
+
++ @Override
+ public int compareTo(Key other) {
+ return compareTo(other, PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME_DEL);
+ }
+
+ @Override
+ public int hashCode() {
+ return WritableComparator.hashBytes(row, row.length) + WritableComparator.hashBytes(colFamily, colFamily.length)
+ + WritableComparator.hashBytes(colQualifier, colQualifier.length) + WritableComparator.hashBytes(colVisibility, colVisibility.length)
+ + (int) (timestamp ^ (timestamp >>> 32));
+ }
+
+ public static String toPrintableString(byte ba[], int offset, int len, int maxLen) {
+ return appendPrintableString(ba, offset, len, maxLen, new StringBuilder()).toString();
+ }
+
+ public static StringBuilder appendPrintableString(byte ba[], int offset, int len, int maxLen, StringBuilder sb) {
+ int plen = Math.min(len, maxLen);
+
+ for (int i = 0; i < plen; i++) {
+ int c = 0xff & ba[offset + i];
+ if (c >= 32 && c <= 126)
+ sb.append((char) c);
+ else
+ sb.append("%" + String.format("%02x;", c));
+ }
+
+ if (len > maxLen) {
+ sb.append("... TRUNCATED");
+ }
+
+ return sb;
+ }
+
+ private StringBuilder rowColumnStringBuilder() {
+ StringBuilder sb = new StringBuilder();
+ appendPrintableString(row, 0, row.length, Constants.MAX_DATA_TO_PRINT, sb);
+ sb.append(" ");
+ appendPrintableString(colFamily, 0, colFamily.length, Constants.MAX_DATA_TO_PRINT, sb);
+ sb.append(":");
+ appendPrintableString(colQualifier, 0, colQualifier.length, Constants.MAX_DATA_TO_PRINT, sb);
+ sb.append(" [");
+ appendPrintableString(colVisibility, 0, colVisibility.length, Constants.MAX_DATA_TO_PRINT, sb);
+ sb.append("]");
+ return sb;
+ }
+
++ @Override
+ public String toString() {
+ StringBuilder sb = rowColumnStringBuilder();
+ sb.append(" ");
+ sb.append(Long.toString(timestamp));
+ sb.append(" ");
+ sb.append(deleted);
+ return sb.toString();
+ }
+
+ public String toStringNoTime() {
+ return rowColumnStringBuilder().toString();
+ }
+
+ /**
+ * Returns the sums of the lengths of the row, column family, column qualifier, and visibility.
+ *
+ * @return row.length + colFamily.length + colQualifier.length + colVisibility.length;
+ */
+ public int getLength() {
+ return row.length + colFamily.length + colQualifier.length + colVisibility.length;
+ }
+
+ /**
+ * Same as {@link #getLength()}.
+ */
+ public int getSize() {
+ return getLength();
+ }
+
+ private static boolean isEqual(byte a1[], byte a2[]) {
+ if (a1 == a2)
+ return true;
+
+ int last = a1.length;
+
+ if (last != a2.length)
+ return false;
+
+ if (last == 0)
+ return true;
+
+ // since sorted data is usually compared in accumulo,
+ // the prefixes will normally be the same... so compare
+ // the last two charachters first.. the most likely place
+ // to have disorder is at end of the strings when the
+ // data is sorted... if those are the same compare the rest
+ // of the data forward... comparing backwards is slower
+ // (compiler and cpu optimized for reading data forward)..
+ // do not want slower comparisons when data is equal...
+ // sorting brings equals data together
+
+ last--;
+
+ if (a1[last] == a2[last]) {
+ for (int i = 0; i < last; i++)
+ if (a1[i] != a2[i])
+ return false;
+ } else {
+ return false;
+ }
+
+ return true;
+
+ }
+
+ /**
+ * Use this to compress a list of keys before sending them via thrift.
+ *
+ * @param param
+ * a list of key/value pairs
+ */
+ public static List<TKeyValue> compress(List<? extends KeyValue> param) {
+
+ List<TKeyValue> tkvl = Arrays.asList(new TKeyValue[param.size()]);
+
+ if (param.size() > 0)
+ tkvl.set(0, new TKeyValue(param.get(0).key.toThrift(), ByteBuffer.wrap(param.get(0).value)));
+
+ for (int i = param.size() - 1; i > 0; i--) {
+ Key prevKey = param.get(i - 1).key;
+ KeyValue kv = param.get(i);
+ Key key = kv.key;
+
+ TKey newKey = null;
+
+ if (isEqual(prevKey.row, key.row)) {
+ newKey = key.toThrift();
+ newKey.row = null;
+ }
+
+ if (isEqual(prevKey.colFamily, key.colFamily)) {
+ if (newKey == null)
+ newKey = key.toThrift();
+ newKey.colFamily = null;
+ }
+
+ if (isEqual(prevKey.colQualifier, key.colQualifier)) {
+ if (newKey == null)
+ newKey = key.toThrift();
+ newKey.colQualifier = null;
+ }
+
+ if (isEqual(prevKey.colVisibility, key.colVisibility)) {
+ if (newKey == null)
+ newKey = key.toThrift();
+ newKey.colVisibility = null;
+ }
+
+ if (newKey == null)
+ newKey = key.toThrift();
+
+ tkvl.set(i, new TKeyValue(newKey, ByteBuffer.wrap(kv.value)));
+ }
+
+ return tkvl;
+ }
+
+ /**
+ * Use this to decompress a list of keys received from thrift.
- *
- * @param param
+ */
-
+ public static void decompress(List<TKeyValue> param) {
+ for (int i = 1; i < param.size(); i++) {
+ TKey prevKey = param.get(i - 1).key;
+ TKey key = param.get(i).key;
+
+ if (key.row == null) {
+ key.row = prevKey.row;
+ }
+ if (key.colFamily == null) {
+ key.colFamily = prevKey.colFamily;
+ }
+ if (key.colQualifier == null) {
+ key.colQualifier = prevKey.colQualifier;
+ }
+ if (key.colVisibility == null) {
+ key.colVisibility = prevKey.colVisibility;
+ }
+ }
+ }
+
+ byte[] getRowBytes() {
+ return row;
+ }
+
+ byte[] getColFamily() {
+ return colFamily;
+ }
+
+ byte[] getColQualifier() {
+ return colQualifier;
+ }
+
+ byte[] getColVisibility() {
+ return colVisibility;
+ }
+
+ public TKey toThrift() {
+ return new TKey(ByteBuffer.wrap(row), ByteBuffer.wrap(colFamily), ByteBuffer.wrap(colQualifier), ByteBuffer.wrap(colVisibility), timestamp);
+ }
+
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ Key r = (Key) super.clone();
+ r.row = Arrays.copyOf(row, row.length);
+ r.colFamily = Arrays.copyOf(colFamily, colFamily.length);
+ r.colQualifier = Arrays.copyOf(colQualifier, colQualifier.length);
+ r.colVisibility = Arrays.copyOf(colVisibility, colVisibility.length);
+ return r;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
index e48914d,0000000..63c594c
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
@@@ -1,783 -1,0 +1,783 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
+/**
+ * keeps track of information needed to identify a tablet
+ * apparently, we only need the endKey and not the start as well
+ *
+ */
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.lang.ref.WeakReference;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.UUID;
+import java.util.WeakHashMap;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.thrift.TKeyExtent;
+import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.hadoop.io.BinaryComparable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+
+public class KeyExtent implements WritableComparable<KeyExtent> {
+
+ private static final WeakHashMap<Text,WeakReference<Text>> tableIds = new WeakHashMap<Text,WeakReference<Text>>();
+
+ private static Text dedupeTableId(Text tableId) {
+ synchronized (tableIds) {
+ WeakReference<Text> etir = tableIds.get(tableId);
+ if (etir != null) {
+ Text eti = etir.get();
+ if (eti != null) {
+ return eti;
+ }
+ }
+
+ tableId = new Text(tableId);
+ tableIds.put(tableId, new WeakReference<Text>(tableId));
+ return tableId;
+ }
+ }
+
+ private Text textTableId;
+ private Text textEndRow;
+ private Text textPrevEndRow;
+
+ private void check() {
+
+ if (getTableId() == null)
+ throw new IllegalArgumentException("null table id not allowed");
+
+ if (getEndRow() == null || getPrevEndRow() == null)
+ return;
+
+ if (getPrevEndRow().compareTo(getEndRow()) >= 0) {
+ throw new IllegalArgumentException("prevEndRow (" + getPrevEndRow() + ") >= endRow (" + getEndRow() + ")");
+ }
+ }
+
+ /**
+ * Default constructor
+ *
+ */
+ public KeyExtent() {
+ this.setTableId(new Text());
+ this.setEndRow(new Text(), false, false);
+ this.setPrevEndRow(new Text(), false, false);
+ }
+
+ public KeyExtent(Text table, Text endRow, Text prevEndRow) {
+ this.setTableId(table);
+ this.setEndRow(endRow, false, true);
+ this.setPrevEndRow(prevEndRow, false, true);
+
+ check();
+ }
+
+ public KeyExtent(KeyExtent extent) {
+ // extent has already deduped table id, so there is no need to do it again
+ this.textTableId = extent.textTableId;
+ this.setEndRow(extent.getEndRow(), false, true);
+ this.setPrevEndRow(extent.getPrevEndRow(), false, true);
+
+ check();
+ }
+
+ public KeyExtent(TKeyExtent tke) {
+ this.setTableId(new Text(ByteBufferUtil.toBytes(tke.table)));
+ this.setEndRow(tke.endRow == null ? null : new Text(ByteBufferUtil.toBytes(tke.endRow)), false, false);
+ this.setPrevEndRow(tke.prevEndRow == null ? null : new Text(ByteBufferUtil.toBytes(tke.prevEndRow)), false, false);
+
+ check();
+ }
+
+ /**
+ * Returns a String representing this extent's entry in the Metadata table
+ *
+ */
+ public Text getMetadataEntry() {
+ return getMetadataEntry(getTableId(), getEndRow());
+ }
+
+ public static Text getMetadataEntry(Text table, Text row) {
+ Text entry = new Text(table);
+
+ if (row == null) {
+ entry.append(new byte[] {'<'}, 0, 1);
+ } else {
+ entry.append(new byte[] {';'}, 0, 1);
+ entry.append(row.getBytes(), 0, row.getLength());
+ }
+
+ return entry;
+
+ }
+
+ // constructor for loading extents from metadata rows
+ public KeyExtent(Text flattenedExtent, Value prevEndRow) {
+ decodeMetadataRow(flattenedExtent);
+
+ // decode the prev row
+ this.setPrevEndRow(decodePrevEndRow(prevEndRow), false, true);
+
+ check();
+ }
+
+ // recreates an encoded extent from a string representation
+ // this encoding is what is stored as the row id of the metadata table
+ public KeyExtent(Text flattenedExtent, Text prevEndRow) {
+
+ decodeMetadataRow(flattenedExtent);
+
+ this.setPrevEndRow(null, false, false);
+ if (prevEndRow != null)
+ this.setPrevEndRow(prevEndRow, false, true);
+
+ check();
+ }
+
+ /**
+ * Sets the extents table id
+ *
+ */
+ public void setTableId(Text tId) {
+
+ if (tId == null)
+ throw new IllegalArgumentException("null table name not allowed");
+
+ this.textTableId = dedupeTableId(tId);
+
+ hashCode = 0;
+ }
+
+ /**
+ * Returns the extent's table id
+ *
+ */
+ public Text getTableId() {
+ return textTableId;
+ }
+
+ private void setEndRow(Text endRow, boolean check, boolean copy) {
+ if (endRow != null)
+ if (copy)
+ this.textEndRow = new Text(endRow);
+ else
+ this.textEndRow = endRow;
+ else
+ this.textEndRow = null;
+
+ hashCode = 0;
+ if (check)
+ check();
+ }
+
+ /**
+ * Sets this extent's end row
+ *
+ */
+ public void setEndRow(Text endRow) {
+ setEndRow(endRow, true, true);
+ }
+
+ /**
+ * Returns this extent's end row
+ *
+ */
+ public Text getEndRow() {
+ return textEndRow;
+ }
+
+ /**
+ * Return the previous extent's end row
+ *
+ */
+ public Text getPrevEndRow() {
+ return textPrevEndRow;
+ }
+
+ private void setPrevEndRow(Text prevEndRow, boolean check, boolean copy) {
+ if (prevEndRow != null)
+ if (copy)
+ this.textPrevEndRow = new Text(prevEndRow);
+ else
+ this.textPrevEndRow = prevEndRow;
+ else
+ this.textPrevEndRow = null;
+
+ hashCode = 0;
+ if (check)
+ check();
+ }
+
+ /**
+ * Sets the previous extent's end row
+ *
+ */
+ public void setPrevEndRow(Text prevEndRow) {
+ setPrevEndRow(prevEndRow, true, true);
+ }
+
+ /**
+ * Populates the extents data fields from a DataInput object
+ *
+ */
++ @Override
+ public void readFields(DataInput in) throws IOException {
+ Text tid = new Text();
+ tid.readFields(in);
+ setTableId(tid);
+ boolean hasRow = in.readBoolean();
+ if (hasRow) {
+ Text er = new Text();
+ er.readFields(in);
+ setEndRow(er, false, false);
+ } else {
+ setEndRow(null, false, false);
+ }
+ boolean hasPrevRow = in.readBoolean();
+ if (hasPrevRow) {
+ Text per = new Text();
+ per.readFields(in);
+ setPrevEndRow(per, false, true);
+ } else {
+ setPrevEndRow((Text) null);
+ }
+
+ hashCode = 0;
+ check();
+ }
+
+ /**
+ * Writes this extent's data fields to a DataOutput object
+ *
+ */
++ @Override
+ public void write(DataOutput out) throws IOException {
+ getTableId().write(out);
+ if (getEndRow() != null) {
+ out.writeBoolean(true);
+ getEndRow().write(out);
+ } else {
+ out.writeBoolean(false);
+ }
+ if (getPrevEndRow() != null) {
+ out.writeBoolean(true);
+ getPrevEndRow().write(out);
+ } else {
+ out.writeBoolean(false);
+ }
+ }
+
+ /**
+ * Returns a String representing the previous extent's entry in the Metadata table
+ *
+ */
+ public Mutation getPrevRowUpdateMutation() {
+ return getPrevRowUpdateMutation(this);
+ }
+
+ /**
+ * Empty start or end rows tell the method there are no start or end rows, and to use all the keyextents that are before the end row if no start row etc.
+ *
+ * @return all the key extents that the rows cover
+ */
+
+ public static Collection<KeyExtent> getKeyExtentsForRange(Text startRow, Text endRow, Set<KeyExtent> kes) {
+ if (kes == null)
+ return Collections.emptyList();
+ if (startRow == null)
+ startRow = new Text();
+ if (endRow == null)
+ endRow = new Text();
+ Collection<KeyExtent> keys = new ArrayList<KeyExtent>();
+ for (KeyExtent ckes : kes) {
+ if (ckes.getPrevEndRow() == null) {
+ if (ckes.getEndRow() == null) {
+ // only tablet
+ keys.add(ckes);
+ } else {
+ // first tablet
+ // if start row = '' then we want everything up to the endRow which will always include the first tablet
+ if (startRow.getLength() == 0) {
+ keys.add(ckes);
+ } else if (ckes.getEndRow().compareTo(startRow) >= 0) {
+ keys.add(ckes);
+ }
+ }
+ } else {
+ if (ckes.getEndRow() == null) {
+ // last tablet
+ // if endRow = '' and we're at the last tablet, add it
+ if (endRow.getLength() == 0) {
+ keys.add(ckes);
+ }
+ if (ckes.getPrevEndRow().compareTo(endRow) < 0) {
+ keys.add(ckes);
+ }
+ } else {
+ // tablet in the middle
+ if (startRow.getLength() == 0) {
+ // no start row
+
+ if (endRow.getLength() == 0) {
+ // no start & end row
+ keys.add(ckes);
+ } else {
+ // just no start row
+ if (ckes.getPrevEndRow().compareTo(endRow) < 0) {
+ keys.add(ckes);
+ }
+ }
+ } else if (endRow.getLength() == 0) {
+ // no end row
+ if (ckes.getEndRow().compareTo(startRow) >= 0) {
+ keys.add(ckes);
+ }
+ } else {
+ // no null prevend or endrows and no empty string start or end rows
+ if ((ckes.getPrevEndRow().compareTo(endRow) < 0 && ckes.getEndRow().compareTo(startRow) >= 0)) {
+ keys.add(ckes);
+ }
+ }
+
+ }
+ }
+ }
+ return keys;
+ }
+
+ public static Text decodePrevEndRow(Value ibw) {
+ Text per = null;
+
+ if (ibw.get()[0] != 0) {
+ per = new Text();
+ per.set(ibw.get(), 1, ibw.get().length - 1);
+ }
+
+ return per;
+ }
+
+ public static Value encodePrevEndRow(Text per) {
+ if (per == null)
+ return new Value(new byte[] {0});
+ byte[] b = new byte[per.getLength() + 1];
+ b[0] = 1;
+ System.arraycopy(per.getBytes(), 0, b, 1, per.getLength());
+ return new Value(b);
+ }
+
+ public static Mutation getPrevRowUpdateMutation(KeyExtent ke) {
+ Mutation m = new Mutation(ke.getMetadataEntry());
+ Constants.METADATA_PREV_ROW_COLUMN.put(m, encodePrevEndRow(ke.getPrevEndRow()));
+ return m;
+ }
+
+ /**
+ * Compares extents based on rows
+ *
+ */
++ @Override
+ public int compareTo(KeyExtent other) {
+
+ int result = getTableId().compareTo(other.getTableId());
+ if (result != 0)
+ return result;
+
+ if (this.getEndRow() == null) {
+ if (other.getEndRow() != null)
+ return 1;
+ } else {
+ if (other.getEndRow() == null)
+ return -1;
+
+ result = getEndRow().compareTo(other.getEndRow());
+ if (result != 0)
+ return result;
+ }
+ if (this.getPrevEndRow() == null) {
+ if (other.getPrevEndRow() == null)
+ return 0;
+ return -1;
+ }
+ if (other.getPrevEndRow() == null)
+ return 1;
+ return this.getPrevEndRow().compareTo(other.getPrevEndRow());
+ }
+
+ private int hashCode = 0;
+
+ @Override
+ public int hashCode() {
+ if (hashCode != 0)
+ return hashCode;
+
+ int prevEndRowHash = 0;
+ int endRowHash = 0;
+ if (this.getEndRow() != null) {
+ endRowHash = this.getEndRow().hashCode();
+ }
+
+ if (this.getPrevEndRow() != null) {
+ prevEndRowHash = this.getPrevEndRow().hashCode();
+ }
+
+ hashCode = getTableId().hashCode() + endRowHash + prevEndRowHash;
+ return hashCode;
+ }
+
+ private boolean equals(Text t1, Text t2) {
+ if (t1 == null || t2 == null)
+ return t1 == t2;
+
+ return t1.equals(t2);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this)
+ return true;
+ if (!(o instanceof KeyExtent))
+ return false;
+ KeyExtent oke = (KeyExtent) o;
+ return textTableId.equals(oke.textTableId) && equals(textEndRow, oke.textEndRow) && equals(textPrevEndRow, oke.textPrevEndRow);
+ }
+
+ @Override
+ public String toString() {
+ String endRowString;
+ String prevEndRowString;
+ String tableIdString = getTableId().toString().replaceAll(";", "\\\\;").replaceAll("\\\\", "\\\\\\\\");
+
+ if (getEndRow() == null)
+ endRowString = "<";
+ else
+ endRowString = ";" + TextUtil.truncate(getEndRow()).toString().replaceAll(";", "\\\\;").replaceAll("\\\\", "\\\\\\\\");
+
+ if (getPrevEndRow() == null)
+ prevEndRowString = "<";
+ else
+ prevEndRowString = ";" + TextUtil.truncate(getPrevEndRow()).toString().replaceAll(";", "\\\\;").replaceAll("\\\\", "\\\\\\\\");
+
+ return tableIdString + endRowString + prevEndRowString;
+ }
+
+ public UUID getUUID() {
+ try {
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+
+ // to get a unique hash it is important to encode the data
+ // like it is being serialized
+
+ this.write(dos);
+
+ dos.close();
+
+ return UUID.nameUUIDFromBytes(baos.toByteArray());
+
+ } catch (IOException e) {
+ // should not happen since we are writing to memory
+ throw new RuntimeException(e);
+ }
+ }
+
+ // note: this is only the encoding of the table id and the last row, not the prev row
+ /**
+ * Populates the extent's fields based on a flatted extent
+ *
+ */
+ private void decodeMetadataRow(Text flattenedExtent) {
+ int semiPos = -1;
+ int ltPos = -1;
+
+ for (int i = 0; i < flattenedExtent.getLength(); i++) {
+ if (flattenedExtent.getBytes()[i] == ';' && semiPos < 0) {
+ // want the position of the first semicolon
+ semiPos = i;
+ }
+
+ if (flattenedExtent.getBytes()[i] == '<') {
+ ltPos = i;
+ }
+ }
+
+ if (semiPos < 0 && ltPos < 0) {
+ throw new IllegalArgumentException("Metadata row does not contain ; or < " + flattenedExtent);
+ }
+
+ if (semiPos < 0) {
+
+ if (ltPos != flattenedExtent.getLength() - 1) {
+ throw new IllegalArgumentException("< must come at end of Metadata row " + flattenedExtent);
+ }
+
+ Text tableId = new Text();
+ tableId.set(flattenedExtent.getBytes(), 0, flattenedExtent.getLength() - 1);
+ this.setTableId(tableId);
+ this.setEndRow(null, false, false);
+ } else {
+
+ Text tableId = new Text();
+ tableId.set(flattenedExtent.getBytes(), 0, semiPos);
+
+ Text endRow = new Text();
+ endRow.set(flattenedExtent.getBytes(), semiPos + 1, flattenedExtent.getLength() - (semiPos + 1));
+
+ this.setTableId(tableId);
+
+ this.setEndRow(endRow, false, false);
+ }
+ }
+
+ public static byte[] tableOfMetadataRow(Text row) {
+ KeyExtent ke = new KeyExtent();
+ ke.decodeMetadataRow(row);
+ return TextUtil.getBytes(ke.getTableId());
+ }
+
+ public boolean contains(final ByteSequence bsrow) {
+ if (bsrow == null) {
+ throw new IllegalArgumentException("Passing null to contains is ambiguous, could be in first or last extent of table");
+ }
+
+ BinaryComparable row = new BinaryComparable() {
+
+ @Override
+ public int getLength() {
+ return bsrow.length();
+ }
+
+ @Override
+ public byte[] getBytes() {
+ if (bsrow.isBackedByArray() && bsrow.offset() == 0)
+ return bsrow.getBackingArray();
+
+ return bsrow.toArray();
+ }
+ };
+
+ if ((this.getPrevEndRow() == null || this.getPrevEndRow().compareTo(row) < 0) && (this.getEndRow() == null || this.getEndRow().compareTo(row) >= 0)) {
+ return true;
+ }
+ return false;
+ }
+
+ public boolean contains(BinaryComparable row) {
+ if (row == null) {
+ throw new IllegalArgumentException("Passing null to contains is ambiguous, could be in first or last extent of table");
+ }
+
+ if ((this.getPrevEndRow() == null || this.getPrevEndRow().compareTo(row) < 0) && (this.getEndRow() == null || this.getEndRow().compareTo(row) >= 0)) {
+ return true;
+ }
+ return false;
+ }
+
+ public Range toDataRange() {
+ return new Range(getPrevEndRow(), false, getEndRow(), true);
+ }
+
+ public Range toMetadataRange() {
+ Text metadataPrevRow = new Text(getTableId());
+ metadataPrevRow.append(new byte[] {';'}, 0, 1);
+ if (getPrevEndRow() != null) {
+ metadataPrevRow.append(getPrevEndRow().getBytes(), 0, getPrevEndRow().getLength());
+ }
+
+ Range range = new Range(metadataPrevRow, getPrevEndRow() == null, getMetadataEntry(), true);
+ return range;
+ }
+
+ public static SortedSet<KeyExtent> findChildren(KeyExtent ke, SortedSet<KeyExtent> tablets) {
+
+ SortedSet<KeyExtent> children = null;
+
+ for (KeyExtent tabletKe : tablets) {
+
+ if (ke.getPrevEndRow() == tabletKe.getPrevEndRow() || ke.getPrevEndRow() != null && tabletKe.getPrevEndRow() != null
+ && tabletKe.getPrevEndRow().compareTo(ke.getPrevEndRow()) == 0) {
+ children = new TreeSet<KeyExtent>();
+ }
+
+ if (children != null) {
+ children.add(tabletKe);
+ }
+
+ if (ke.getEndRow() == tabletKe.getEndRow() || ke.getEndRow() != null && tabletKe.getEndRow() != null
+ && tabletKe.getEndRow().compareTo(ke.getEndRow()) == 0) {
+ return children;
+ }
+ }
+
+ return new TreeSet<KeyExtent>();
+ }
+
+ public static KeyExtent findContainingExtent(KeyExtent extent, SortedSet<KeyExtent> extents) {
+
+ KeyExtent lookupExtent = new KeyExtent(extent);
+ lookupExtent.setPrevEndRow((Text) null);
+
+ SortedSet<KeyExtent> tailSet = extents.tailSet(lookupExtent);
+
+ if (tailSet.isEmpty()) {
+ return null;
+ }
+
+ KeyExtent first = tailSet.first();
+
+ if (first.getTableId().compareTo(extent.getTableId()) != 0) {
+ return null;
+ }
+
+ if (first.getPrevEndRow() == null) {
+ return first;
+ }
+
+ if (extent.getPrevEndRow() == null) {
+ return null;
+ }
+
+ if (extent.getPrevEndRow().compareTo(first.getPrevEndRow()) >= 0)
+ return first;
+ return null;
+ }
+
+ private static boolean startsAfter(KeyExtent nke, KeyExtent ke) {
+
+ int tiCmp = ke.getTableId().compareTo(nke.getTableId());
+
+ if (tiCmp > 0) {
+ return true;
+ }
+
+ return ke.getPrevEndRow() != null && nke.getEndRow() != null && ke.getPrevEndRow().compareTo(nke.getEndRow()) >= 0;
+ }
+
+ private static Text rowAfterPrevRow(KeyExtent nke) {
+ Text row = new Text(nke.getPrevEndRow());
+ row.append(new byte[] {0}, 0, 1);
+ return row;
+ }
+
+ // Some duplication with TabletLocatorImpl
+ public static Set<KeyExtent> findOverlapping(KeyExtent nke, SortedSet<KeyExtent> extents) {
+ if (nke == null || extents == null || extents.isEmpty())
+ return Collections.emptySet();
+
+ SortedSet<KeyExtent> start;
+
+ if (nke.getPrevEndRow() != null) {
+ Text row = rowAfterPrevRow(nke);
+ KeyExtent lookupKey = new KeyExtent(nke.getTableId(), row, null);
+ start = extents.tailSet(lookupKey);
+ } else {
+ KeyExtent lookupKey = new KeyExtent(nke.getTableId(), new Text(), null);
+ start = extents.tailSet(lookupKey);
+ }
+
+ TreeSet<KeyExtent> result = new TreeSet<KeyExtent>();
+ for (KeyExtent ke : start) {
+ if (startsAfter(nke, ke)) {
+ break;
+ }
+ result.add(ke);
+ }
+ return result;
+ }
+
+ public boolean overlaps(KeyExtent other) {
+ SortedSet<KeyExtent> set = new TreeSet<KeyExtent>();
+ set.add(other);
+ return !findOverlapping(this, set).isEmpty();
+ }
+
+ // Specialization of findOverlapping(KeyExtent, SortedSet<KeyExtent> to work with SortedMap
+ public static Set<KeyExtent> findOverlapping(KeyExtent nke, SortedMap<KeyExtent,? extends Object> extents) {
+ if (nke == null || extents == null || extents.isEmpty())
+ return Collections.emptySet();
+
+ SortedMap<KeyExtent,? extends Object> start;
+
+ if (nke.getPrevEndRow() != null) {
+ Text row = rowAfterPrevRow(nke);
+ KeyExtent lookupKey = new KeyExtent(nke.getTableId(), row, null);
+ start = extents.tailMap(lookupKey);
+ } else {
+ KeyExtent lookupKey = new KeyExtent(nke.getTableId(), new Text(), null);
+ start = extents.tailMap(lookupKey);
+ }
+
+ TreeSet<KeyExtent> result = new TreeSet<KeyExtent>();
+ for (Entry<KeyExtent,? extends Object> entry : start.entrySet()) {
+ KeyExtent ke = entry.getKey();
+ if (startsAfter(nke, ke)) {
+ break;
+ }
+ result.add(ke);
+ }
+ return result;
+ }
+
+ public static Text getMetadataEntry(KeyExtent extent) {
+ return getMetadataEntry(extent.getTableId(), extent.getEndRow());
+ }
+
+ public TKeyExtent toThrift() {
+ return new TKeyExtent(TextUtil.getByteBuffer(textTableId), textEndRow == null ? null : TextUtil.getByteBuffer(textEndRow), textPrevEndRow == null ? null
+ : TextUtil.getByteBuffer(textPrevEndRow));
+ }
+
- /**
- * @param prevExtent
- */
+ public boolean isPreviousExtent(KeyExtent prevExtent) {
+ if (prevExtent == null)
+ return getPrevEndRow() == null;
+
+ if (!prevExtent.getTableId().equals(getTableId()))
+ throw new IllegalArgumentException("Cannot compare accross tables " + prevExtent + " " + this);
+
+ if (prevExtent.getEndRow() == null)
+ return false;
+
+ if (getPrevEndRow() == null)
+ return false;
+
+ return prevExtent.getEndRow().equals(getPrevEndRow());
+ }
+
+ public boolean isMeta() {
+ return getTableId().toString().equals(Constants.METADATA_TABLE_ID);
+ }
+
+ public boolean isRootTablet() {
+ return this.compareTo(Constants.ROOT_TABLET_EXTENT) == 0;
+ }
+}
[55/64] [abbrv] Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleParameters.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleParameters.java
index 5ae072a,0000000..244a877
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleParameters.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleParameters.java
@@@ -1,643 -1,0 +1,637 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.core.security.crypto;
+
+import java.io.FilterOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.security.SecureRandom;
+import java.util.Map;
+
+import javax.crypto.Cipher;
+import javax.crypto.CipherOutputStream;
+
+/**
+ * This class defines several parameters needed by by a module providing cryptographic stream support in Accumulo. The following Javadoc details which
+ * parameters are used for which operations (encryption vs. decryption), which ones return values (i.e. are "out" parameters from the {@link CryptoModule}), and
+ * which ones are required versus optional in certain situations.
+ *
+ * Most of the time, these classes can be constructed using
+ * {@link CryptoModuleFactory#createParamsObjectFromAccumuloConfiguration(org.apache.accumulo.core.conf.AccumuloConfiguration)}.
+ */
+public class CryptoModuleParameters {
+
+ /**
+ * Gets the name of the symmetric algorithm to use for encryption.
+ *
+ * @see CryptoModuleParameters#setAlgorithmName(String)
+ */
+
+ public String getAlgorithmName() {
+ return algorithmName;
+ }
+
+ /**
+ * Sets the name of the symmetric algorithm to use for an encryption stream.
+ * <p>
+ * Valid names are names recognized by your cryptographic engine provider. For the default Java provider, valid names would include things like "AES", "RC4",
+ * "DESede", etc.
+ * <p>
+ * For <b>encryption</b>, this value is <b>required</b> and is always used. Its value should be prepended or otherwise included with the ciphertext for future
+ * decryption. <br>
+ * For <b>decryption</b>, this value is often disregarded in favor of the value encoded with the ciphertext.
+ *
+ * @param algorithmName
+ * the name of the cryptographic algorithm to use.
+ * @see <a href="http://docs.oracle.com/javase/1.5.0/docs/guide/security/jce/JCERefGuide.html#AppA">Standard Algorithm Names in JCE</a>
+ *
+ */
+
+ public void setAlgorithmName(String algorithmName) {
+ this.algorithmName = algorithmName;
+ }
+
+ /**
+ * Gets the name of the encryption mode to use for encryption.
+ *
+ * @see CryptoModuleParameters#setEncryptionMode(String)
+ */
+
+ public String getEncryptionMode() {
+ return encryptionMode;
+ }
+
+ /**
+ * Sets the name of the encryption mode to use for an encryption stream.
+ * <p>
+ * Valid names are names recognized by your cryptographic engine provider. For the default Java provider, valid names would include things like "EBC", "CBC",
+ * "CFB", etc.
+ * <p>
+ * For <b>encryption</b>, this value is <b>required</b> and is always used. Its value should be prepended or otherwise included with the ciphertext for future
+ * decryption. <br>
+ * For <b>decryption</b>, this value is often disregarded in favor of the value encoded with the ciphertext.
+ *
+ * @param encryptionMode
+ * the name of the encryption mode to use.
+ * @see <a href="http://docs.oracle.com/javase/1.5.0/docs/guide/security/jce/JCERefGuide.html#AppA">Standard Mode Names in JCE</a>
+ *
+ */
+
+ public void setEncryptionMode(String encryptionMode) {
+ this.encryptionMode = encryptionMode;
+ }
+
+ /**
+ * Gets the name of the padding type to use for encryption.
+ *
+ * @see CryptoModuleParameters#setPadding(String)
+ */
+
+ public String getPadding() {
+ return padding;
+ }
+
+ /**
+ * Sets the name of the padding type to use for an encryption stream.
+ * <p>
+ * Valid names are names recognized by your cryptographic engine provider. For the default Java provider, valid names would include things like "NoPadding",
+ * "None", etc.
+ * <p>
+ * For <b>encryption</b>, this value is <b>required</b> and is always used. Its value should be prepended or otherwise included with the ciphertext for future
+ * decryption. <br>
+ * For <b>decryption</b>, this value is often disregarded in favor of the value encoded with the ciphertext.
+ *
+ * @param padding
+ * the name of the padding type to use.
+ * @see <a href="http://docs.oracle.com/javase/1.5.0/docs/guide/security/jce/JCERefGuide.html#AppA">Standard Padding Names in JCE</a>
+ *
+ */
+ public void setPadding(String padding) {
+ this.padding = padding;
+ }
+
+ /**
+ * Gets the plaintext secret key.
+ * <p>
+ * For <b>decryption</b>, this value is often the out parameter of using a secret key encryption strategy to decrypt an encrypted version of this secret key.
+ * (See {@link CryptoModuleParameters#setKeyEncryptionStrategyClass(String)}.)
+ *
+ *
+ * @see CryptoModuleParameters#setPlaintextKey(byte[])
+ */
+ public byte[] getPlaintextKey() {
+ return plaintextKey;
+ }
+
+ /**
+ * Sets the plaintext secret key that will be used to encrypt and decrypt bytes.
+ * <p>
+ * Valid values and lengths for this secret key depend entirely on the algorithm type. Refer to the documentation about the algorithm for further information.
+ * <p>
+ * For <b>encryption</b>, this value is <b>optional</b>. If it is not provided, it will be automatically generated by the underlying cryptographic module. <br>
+ * For <b>decryption</b>, this value is often obtained from the underlying cipher stream, or derived from the encrypted version of the key (see
+ * {@link CryptoModuleParameters#setEncryptedKey(byte[])}).
+ *
+ * @param plaintextKey
+ * the value of the plaintext secret key
+ */
+
+ public void setPlaintextKey(byte[] plaintextKey) {
+ this.plaintextKey = plaintextKey;
+ }
+
+ /**
+ * Gets the length of the secret key.
+ *
+ * @see CryptoModuleParameters#setKeyLength(int)
+ */
+ public int getKeyLength() {
+ return keyLength;
+ }
+
+ /**
+ * Sets the length of the secret key that will be used to encrypt and decrypt bytes.
+ * <p>
+ * Valid lengths depend entirely on the algorithm type. Refer to the documentation about the algorithm for further information. (For example, AES may use
+ * either 128 or 256 bit keys in the default Java cryptography provider.)
+ * <p>
+ * For <b>encryption</b>, this value is <b>required if the secret key is not set</b>. <br>
+ * For <b>decryption</b>, this value is often obtained from the underlying cipher stream, or derived from the encrypted version of the key (see
+ * {@link CryptoModuleParameters#setEncryptedKey(byte[])}).
+ *
+ * @param keyLength
+ * the length of the secret key to be generated
+ */
+
+ public void setKeyLength(int keyLength) {
+ this.keyLength = keyLength;
+ }
+
+ /**
+ * Gets the random number generator name.
+ *
+ * @see CryptoModuleParameters#setRandomNumberGenerator(String)
+ */
+
+ public String getRandomNumberGenerator() {
+ return randomNumberGenerator;
+ }
+
+ /**
+ * Sets the name of the random number generator to use. The default for this for the baseline JCE implementation is "SHA1PRNG".
+ * <p>
+ *
+ * <p>
+ * For <b>encryption</b>, this value is <b>required</b>. <br>
+ * For <b>decryption</b>, this value is often obtained from the underlying cipher stream.
+ *
+ * @param randomNumberGenerator
+ * the name of the random number generator to use
+ */
+
+ public void setRandomNumberGenerator(String randomNumberGenerator) {
+ this.randomNumberGenerator = randomNumberGenerator;
+ }
+
+ /**
+ * Gets the random number generator provider name.
+ *
+ * @see CryptoModuleParameters#setRandomNumberGeneratorProvider(String)
+ */
+ public String getRandomNumberGeneratorProvider() {
+ return randomNumberGeneratorProvider;
+ }
+
+ /**
+ * Sets the name of the random number generator provider to use. The default for this for the baseline JCE implementation is "SUN".
+ * <p>
+ * The provider, as the name implies, provides the RNG implementation specified by {@link CryptoModuleParameters#getRandomNumberGenerator()}.
+ * <p>
+ * For <b>encryption</b>, this value is <b>required</b>. <br>
+ * For <b>decryption</b>, this value is often obtained from the underlying cipher stream.
+ *
+ * @param randomNumberGeneratorProvider
+ * the name of the provider to use
+ */
+
+ public void setRandomNumberGeneratorProvider(String randomNumberGeneratorProvider) {
+ this.randomNumberGeneratorProvider = randomNumberGeneratorProvider;
+ }
+
+ /**
+ * Gets the key encryption strategy class.
+ *
+ * @see CryptoModuleParameters#setKeyEncryptionStrategyClass(String)
+ */
+
+ public String getKeyEncryptionStrategyClass() {
+ return keyEncryptionStrategyClass;
+ }
+
+ /**
+ * Sets the class name of the key encryption strategy class. The class obeys the {@link SecretKeyEncryptionStrategy} interface. It instructs the
+ * {@link DefaultCryptoModule} on how to encrypt the keys it uses to secure the streams.
+ * <p>
+ * The default implementation of this interface, {@link CachingHDFSSecretKeyEncryptionStrategy}, creates a random key encryption key (KEK) as another symmetric
+ * key and places the KEK into HDFS. <i>This is not really very secure.</i> Users of the crypto modules are encouraged to either safeguard that KEK carefully
+ * or to obtain and use another {@link SecretKeyEncryptionStrategy} class.
+ * <p>
+ * For <b>encryption</b>, this value is <b>optional</b>. If it is not specified, then it assumed that the secret keys used for encrypting files will not be
+ * encrypted. This is not a secure approach, thus setting this is highly recommended.<br>
+ * For <b>decryption</b>, this value is often obtained from the underlying cipher stream. However, the underlying stream's value can be overridden (at least
+ * when using {@link DefaultCryptoModule}) by setting the {@link CryptoModuleParameters#setOverrideStreamsSecretKeyEncryptionStrategy(boolean)} to true.
+ *
+ * @param keyEncryptionStrategyClass
+ * the name of the key encryption strategy class to use
+ */
+ public void setKeyEncryptionStrategyClass(String keyEncryptionStrategyClass) {
+ this.keyEncryptionStrategyClass = keyEncryptionStrategyClass;
+ }
+
+ /**
+ * Gets the encrypted version of the plaintext key. This parameter is generally either obtained from an underlying stream or computed in the process of
+ * employed the {@link CryptoModuleParameters#getKeyEncryptionStrategyClass()}.
+ *
+ * @see CryptoModuleParameters#setEncryptedKey(byte[])
+ */
+ public byte[] getEncryptedKey() {
+ return encryptedKey;
+ }
+
+ /**
+ * Sets the encrypted version of the plaintext key ({@link CryptoModuleParameters#getPlaintextKey()}). Generally this operation will be done either by:
+ * <p>
+ * <ul>
+ * <li>the code reading an encrypted stream and coming across the encrypted version of one of these keys, OR
+ * <li>the {@link CryptoModuleParameters#getKeyEncryptionStrategyClass()} that encrypted the plaintext key (see
+ * {@link CryptoModuleParameters#getPlaintextKey()}).
+ * <ul>
+ * <p>
+ * For <b>encryption</b>, this value is generally not required, but is usually set by the underlying module during encryption. <br>
+ * For <b>decryption</b>, this value is <b>usually required</b>.
+ *
+ *
+ * @param encryptedKey
+ * the encrypted value of the plaintext key
+ */
+
+ public void setEncryptedKey(byte[] encryptedKey) {
+ this.encryptedKey = encryptedKey;
+ }
+
+ /**
+ * Gets the opaque ID associated with the encrypted version of the plaintext key.
+ *
+ * @see CryptoModuleParameters#setOpaqueKeyEncryptionKeyID(String)
+ */
+ public String getOpaqueKeyEncryptionKeyID() {
+ return opaqueKeyEncryptionKeyID;
+ }
+
+ /**
+ * Sets an opaque ID assocaited with the encrypted version of the plaintext key.
+ * <p>
+ * Often, implementors of the {@link SecretKeyEncryptionStrategy} will need to record some information about how they encrypted a particular plaintext key.
+ * For example, if the strategy employs several keys for its encryption, it will want to record which key it used. The caller should not have to worry about
+ * the format or contents of this internal ID; thus, the strategy class will encode whatever information it needs into this string. It is then beholden to the
+ * calling code to record this opqaue string properly to the underlying cryptographically-encoded stream, and then set the opaque ID back into this parameter
+ * object upon reading.
+ * <p>
+ * For <b>encryption</b>, this value is generally not required, but will be typically generated and set by the {@link SecretKeyEncryptionStrategy} class (see
+ * {@link CryptoModuleParameters#getKeyEncryptionStrategyClass()}). <br>
+ * For <b>decryption</b>, this value is <b>required</b>, though it will typically be read from the underlying stream.
+ *
+ * @param opaqueKeyEncryptionKeyID
+ * the opaque ID assoicated with the encrypted version of the plaintext key (see {@link CryptoModuleParameters#getEncryptedKey()}).
+ */
+
+ public void setOpaqueKeyEncryptionKeyID(String opaqueKeyEncryptionKeyID) {
+ this.opaqueKeyEncryptionKeyID = opaqueKeyEncryptionKeyID;
+ }
+
+ /**
+ * Gets the flag that indicates whether or not the module should record its cryptographic parameters to the stream automatically, or rely on the calling code
+ * to do so.
+ *
+ * @see CryptoModuleParameters#setRecordParametersToStream(boolean)
+ */
+ public boolean getRecordParametersToStream() {
+ return recordParametersToStream;
+ }
+
+ /**
+ * Gets the flag that indicates whether or not the module should record its cryptographic parameters to the stream automatically, or rely on the calling code
+ * to do so.
+ *
+ * <p>
+ *
+ * If this is set to <i>true</i>, then the stream passed to {@link CryptoModule#getEncryptingOutputStream(CryptoModuleParameters)} will be <i>written to by
+ * the module</i> before it is returned to the caller. There are situations where it is easier to let the crypto module do this writing on behalf of the
+ * caller, and other times where it is not appropriate (if the format of the underlying stream must be carefully maintained, for instance).
+ *
+ * @param recordParametersToStream
+ * whether or not to require the module to record its parameters to the stream by itself
+ */
+ public void setRecordParametersToStream(boolean recordParametersToStream) {
+ this.recordParametersToStream = recordParametersToStream;
+ }
+
+ /**
+ * Gets the flag that indicates whether or not to close the underlying stream when the cipher stream is closed.
+ *
+ * @see CryptoModuleParameters#setCloseUnderylingStreamAfterCryptoStreamClose(boolean)
+ */
+ public boolean getCloseUnderylingStreamAfterCryptoStreamClose() {
+ return closeUnderylingStreamAfterCryptoStreamClose;
+ }
+
+ /**
+ * Sets the flag that indicates whether or not to close the underlying stream when the cipher stream is closed.
+ *
+ * <p>
+ *
+ * {@link CipherOutputStream} will only output its padding bytes when its {@link CipherOutputStream#close()} method is called. However, there are times when a
+ * caller doesn't want its underlying stream closed at the time that the {@link CipherOutputStream} is closed. This flag indicates that the
+ * {@link CryptoModule} should wrap the underlying stream in a basic {@link FilterOutputStream} which will swallow any close() calls and prevent them from
+ * propogating to the underlying stream.
+ *
+ * @param closeUnderylingStreamAfterCryptoStreamClose
+ * the flag that indicates whether or not to close the underlying stream when the cipher stream is closed
+ */
+ public void setCloseUnderylingStreamAfterCryptoStreamClose(boolean closeUnderylingStreamAfterCryptoStreamClose) {
+ this.closeUnderylingStreamAfterCryptoStreamClose = closeUnderylingStreamAfterCryptoStreamClose;
+ }
+
+ /**
+ * Gets the flag that indicates if the underlying stream's key encryption strategy should be overridden by the currently configured key encryption strategy.
+ *
+ * @see CryptoModuleParameters#setOverrideStreamsSecretKeyEncryptionStrategy(boolean)
+ */
+ public boolean getOverrideStreamsSecretKeyEncryptionStrategy() {
+ return overrideStreamsSecretKeyEncryptionStrategy;
+ }
+
+ /**
+ * Sets the flag that indicates if the underlying stream's key encryption strategy should be overridden by the currently configured key encryption strategy.
+ *
+ * <p>
+ *
+ * So, why is this important? Say you started out with the default secret key encryption strategy. So, now you have a secret key in HDFS that encrypts all the
+ * other secret keys. <i>Then</i> you deploy a key management solution. You want to move that secret key up to the key management server. Great! No problem.
+ * Except, all your encrypted files now contain a setting that says
+ * "hey I was encrypted by the default strategy, so find decrypt my key using that, not the key management server". This setting signals the
+ * {@link CryptoModule} that it should ignore the setting in the file and prefer the one from the configuration.
+ *
+ * @param overrideStreamsSecretKeyEncryptionStrategy
+ * the flag that indicates if the underlying stream's key encryption strategy should be overridden by the currently configured key encryption
+ * strategy
+ */
+
+ public void setOverrideStreamsSecretKeyEncryptionStrategy(boolean overrideStreamsSecretKeyEncryptionStrategy) {
+ this.overrideStreamsSecretKeyEncryptionStrategy = overrideStreamsSecretKeyEncryptionStrategy;
+ }
+
+ /**
+ * Gets the plaintext output stream to wrap for encryption.
+ *
+ * @see CryptoModuleParameters#setPlaintextOutputStream(OutputStream)
+ */
+ public OutputStream getPlaintextOutputStream() {
+ return plaintextOutputStream;
+ }
+
+ /**
+ * Sets the plaintext output stream to wrap for encryption.
+ *
+ * <p>
+ *
+ * For <b>encryption</b>, this parameter is <b>required</b>. <br>
+ * For <b>decryption</b>, this parameter is ignored.
- *
- * @param plaintextOutputStream
+ */
+ public void setPlaintextOutputStream(OutputStream plaintextOutputStream) {
+ this.plaintextOutputStream = plaintextOutputStream;
+ }
+
+ /**
+ * Gets the encrypted output stream, which is nearly always a wrapped version of the output stream from
+ * {@link CryptoModuleParameters#getPlaintextOutputStream()}.
+ *
+ * <p>
+ *
+ * Generally this method is used by {@link CryptoModule} classes as an <i>out</i> parameter from calling
+ * {@link CryptoModule#getEncryptingOutputStream(CryptoModuleParameters)}.
+ *
+ * @see CryptoModuleParameters#setEncryptedOutputStream(OutputStream)
+ */
+
+ public OutputStream getEncryptedOutputStream() {
+ return encryptedOutputStream;
+ }
+
+ /**
+ * Sets the encrypted output stream. This method should really only be called by {@link CryptoModule} implementations unless something very unusual is going
+ * on.
+ *
+ * @param encryptedOutputStream
+ * the encrypted version of the stream from output stream from {@link CryptoModuleParameters#getPlaintextOutputStream()}.
+ */
+ public void setEncryptedOutputStream(OutputStream encryptedOutputStream) {
+ this.encryptedOutputStream = encryptedOutputStream;
+ }
+
+ /**
+ * Gets the plaintext input stream, which is nearly always a wrapped version of the output from {@link CryptoModuleParameters#getEncryptedInputStream()}.
+ *
+ * <p>
+ *
+ * Generally this method is used by {@link CryptoModule} classes as an <i>out</i> parameter from calling
+ * {@link CryptoModule#getDecryptingInputStream(CryptoModuleParameters)}.
+ *
+ *
+ * @see CryptoModuleParameters#setPlaintextInputStream(InputStream)
+ */
+ public InputStream getPlaintextInputStream() {
+ return plaintextInputStream;
+ }
+
+ /**
+ * Sets the plaintext input stream, which is nearly always a wrapped version of the output from {@link CryptoModuleParameters#getEncryptedInputStream()}.
+ *
+ * <p>
+ *
+ * This method should really only be called by {@link CryptoModule} implementations.
- *
- * @param plaintextInputStream
+ */
+
+ public void setPlaintextInputStream(InputStream plaintextInputStream) {
+ this.plaintextInputStream = plaintextInputStream;
+ }
+
+ /**
+ * Gets the encrypted input stream to wrap for decryption.
+ *
+ * @see CryptoModuleParameters#setEncryptedInputStream(InputStream)
+ */
+ public InputStream getEncryptedInputStream() {
+ return encryptedInputStream;
+ }
+
+ /**
+ * Sets the encrypted input stream to wrap for decryption.
- *
- * @param encryptedInputStream
+ */
+
+ public void setEncryptedInputStream(InputStream encryptedInputStream) {
+ this.encryptedInputStream = encryptedInputStream;
+ }
+
+ /**
+ * Gets the initialized cipher object.
+ *
+ *
+ * @see CryptoModuleParameters#setCipher(Cipher)
+ */
+ public Cipher getCipher() {
+ return cipher;
+ }
+
+ /**
+ * Sets the initialized cipher object. Generally speaking, callers do not have to create and set this object. There may be circumstances where the cipher
+ * object is created outside of the module (to determine IV lengths, for one). If it is created and you want the module to use the cipher you already
+ * initialized, set it here.
+ *
+ * @param cipher
+ * the cipher object
+ */
+ public void setCipher(Cipher cipher) {
+ this.cipher = cipher;
+ }
+
+ /**
+ * Gets the initialized secure random object.
+ *
+ * @see CryptoModuleParameters#setSecureRandom(SecureRandom)
+ */
+ public SecureRandom getSecureRandom() {
+ return secureRandom;
+ }
+
+ /**
+ * Sets the initialized secure random object. Generally speaking, callers do not have to create and set this object. There may be circumstances where the
+ * random object is created outside of the module (for instance, to create a random secret key). If it is created outside the module and you want the module
+ * to use the random object you already created, set it here.
+ *
+ * @param secureRandom
+ * the {@link SecureRandom} object
+ */
+
+ public void setSecureRandom(SecureRandom secureRandom) {
+ this.secureRandom = secureRandom;
+ }
+
+ /**
+ * Gets the initialization vector to use for this crypto module.
+ *
+ * @see CryptoModuleParameters#setInitializationVector(byte[])
+ */
+ public byte[] getInitializationVector() {
+ return initializationVector;
+ }
+
+ /**
+ * Sets the initialization vector to use for this crypto module.
+ *
+ * <p>
+ *
+ * For <b>encryption</b>, this parameter is <i>optional</i>. If the initialization vector is created by the caller, for whatever reasons, it can be set here
+ * and the crypto module will use it. <br>
+ *
+ * For <b>decryption</b>, this parameter is <b>required</b>. It should be read from the underlying stream that contains the encrypted data.
+ *
+ * @param initializationVector
+ * the initialization vector to use for this crypto operation.
+ */
+ public void setInitializationVector(byte[] initializationVector) {
+ this.initializationVector = initializationVector;
+ }
+
+ /**
+ * Gets the size of the buffering stream that sits above the cipher stream
+ */
+ public int getBlockStreamSize() {
+ return blockStreamSize;
+ }
+
+ /**
+ * Sets the size of the buffering stream that sits above the cipher stream
+ */
+ public void setBlockStreamSize(int blockStreamSize) {
+ this.blockStreamSize = blockStreamSize;
+ }
+
+ /**
+ * Gets the overall set of options for the {@link CryptoModule}.
+ *
+ * @see CryptoModuleParameters#setAllOptions(Map)
+ */
+ public Map<String,String> getAllOptions() {
+ return allOptions;
+ }
+
+ /**
+ * Sets the overall set of options for the {@link CryptoModule}.
+ *
+ * <p>
+ *
+ * Often, options for the cryptographic modules will be encoded as key/value pairs in a configuration file. This map represents those values. It may include
+ * some of the parameters already called out as members of this class. It may contain any number of additional parameters which may be required by different
+ * module or key encryption strategy implementations.
+ *
+ * @param allOptions
+ * the set of key/value pairs that confiure a module, based on a configuration file
+ */
+ public void setAllOptions(Map<String,String> allOptions) {
+ this.allOptions = allOptions;
+ }
+
+ private String algorithmName = null;
+ private String encryptionMode = null;
+ private String padding = null;
+ private byte[] plaintextKey;
+ private int keyLength = 0;
+ private String randomNumberGenerator = null;
+ private String randomNumberGeneratorProvider = null;
+
+ private String keyEncryptionStrategyClass;
+ private byte[] encryptedKey;
+ private String opaqueKeyEncryptionKeyID;
+
+ private boolean recordParametersToStream = true;
+ private boolean closeUnderylingStreamAfterCryptoStreamClose = true;
+ private boolean overrideStreamsSecretKeyEncryptionStrategy = false;
+
+ private OutputStream plaintextOutputStream;
+ private OutputStream encryptedOutputStream;
+ private InputStream plaintextInputStream;
+ private InputStream encryptedInputStream;
+
+ private Cipher cipher;
+ private SecureRandom secureRandom;
+ private byte[] initializationVector;
+
+ private Map<String,String> allOptions;
+ private int blockStreamSize;
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/test/java/org/apache/accumulo/core/client/lexicoder/ReverseLexicoderTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/lexicoder/ReverseLexicoderTest.java
index cc363c7,0000000..e6bfca8
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/lexicoder/ReverseLexicoderTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/lexicoder/ReverseLexicoderTest.java
@@@ -1,62 -1,0 +1,60 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.lexicoder;
+
+import java.io.UnsupportedEncodingException;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+
+import org.junit.Test;
+
+public class ReverseLexicoderTest extends LexicoderTest {
+ public void testSortOrder() {
+ Comparator<Long> comp = Collections.reverseOrder();
+ assertSortOrder(new ReverseLexicoder<Long>(new LongLexicoder()), comp, Long.MIN_VALUE, 0xff1234567890abcdl, 0xffff1234567890abl, 0xffffff567890abcdl,
+ 0xffffffff7890abcdl, 0xffffffffff90abcdl, 0xffffffffffffabcdl, 0xffffffffffffffcdl, -1l, 0l, 0x01l, 0x1234l, 0x123456l, 0x12345678l, 0x1234567890l,
+ 0x1234567890abl, 0x1234567890abcdl, 0x1234567890abcdefl, Long.MAX_VALUE);
+
+ Comparator<String> comp2 = Collections.reverseOrder();
+ assertSortOrder(new ReverseLexicoder<String>(new StringLexicoder()), comp2, "a", "aa", "ab", "b", "aab");
+
+ }
+
+ /**
+ * Just a simple test verifying reverse indexed dates
- *
- * @throws UnsupportedEncodingException
+ */
+ @Test
+ public void testReverseSortDates() throws UnsupportedEncodingException {
+
+ ReverseLexicoder<Date> revLex = new ReverseLexicoder<Date>(new DateLexicoder());
+
+ Date date1 = new Date();
+ Date date2 = new Date(System.currentTimeMillis() + 10000);
+ Date date3 = new Date(System.currentTimeMillis() + 500);
+
+ Comparator<Date> comparator = Collections.reverseOrder();
+ assertSortOrder(revLex, comparator, date1, date2, date3);
+
+ // truncate date to hours
+ long time = System.currentTimeMillis() - (System.currentTimeMillis() % 3600000);
+ Date date = new Date(time);
+
+ System.out.println(date);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
index e83453e,3ec9bb1..13490e0
--- a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
@@@ -57,23 -55,9 +57,21 @@@ public class AccumuloInputFormatTest
private static final String PREFIX = AccumuloInputFormatTest.class.getSimpleName();
private static final String INSTANCE_NAME = PREFIX + "_mapred_instance";
private static final String TEST_TABLE_1 = PREFIX + "_mapred_table_1";
-
+
+ private JobConf job;
+
+ @BeforeClass
+ public static void setupClass() {
+ System.setProperty("hadoop.tmp.dir", System.getProperty("user.dir") + "/target/hadoop-tmp");
+ }
+
+ @Before
+ public void createJob() {
+ job = new JobConf();
+ }
+
/**
* Check that the iterator configuration is getting stored in the Job conf correctly.
- *
- * @throws IOException
*/
@Test
public void testSetIterator() throws IOException {
@@@ -152,11 -141,9 +150,9 @@@
assertEquals(list.get(1).getOptions().get(key), value);
assertEquals(list.get(1).getOptions().get(key + "2"), value);
}
-
+
/**
* Test getting iterator settings for multiple iterators set
- *
- * @throws IOException
*/
@Test
public void testGetIteratorSettings() throws IOException {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
index 54bd127,ae5e395..2500972
--- a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
@@@ -64,9 -64,45 +64,7 @@@ public class AccumuloInputFormatTest
private static final String PREFIX = AccumuloInputFormatTest.class.getSimpleName();
/**
- * Test basic setting & getting of max versions.
- *
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- @Deprecated
- @Test
- public void testMaxVersions() throws IOException {
- Job job = new Job();
- AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 1);
- int version = AccumuloInputFormat.getMaxVersions(job.getConfiguration());
- assertEquals(1, version);
- }
-
- /**
- * Test max versions with an invalid value.
- *
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- @Deprecated
- @Test(expected = IOException.class)
- public void testMaxVersionsLessThan1() throws IOException {
- Job job = new Job();
- AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 0);
- }
-
- /**
- * Test no max version configured.
- */
- @Deprecated
- @Test
- public void testNoMaxVersion() throws IOException {
- Job job = new Job();
- assertEquals(-1, AccumuloInputFormat.getMaxVersions(job.getConfiguration()));
- }
-
- /**
* Check that the iterator configuration is getting stored in the Job conf correctly.
- *
- * @throws IOException
*/
@Test
public void testSetIterator() throws IOException {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
index 009be17,0000000..c06df51
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
@@@ -1,317 -1,0 +1,309 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.core.client.mock;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Random;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.NamespaceNotEmptyException;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.NamespaceOperations;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.Filter;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.security.Authorizations;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class MockNamespacesTest {
+
+ Random random = new Random();
+ public static TemporaryFolder folder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
+
+ /**
+ * This test creates a table without specifying a namespace. In this case, it puts the table into the default namespace.
- *
- * @throws Exception
+ */
+ @Test
+ public void testDefaultNamespace() throws Exception {
+ String tableName = "test";
+ Instance instance = new MockInstance("default");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ assertTrue(c.namespaceOperations().exists(Namespaces.DEFAULT_NAMESPACE));
+ c.tableOperations().create(tableName);
+ assertTrue(c.tableOperations().exists(tableName));
+ }
+
+ /**
+ * This test creates a new namespace "testing" and a table "testing.table1" which puts "table1" into the "testing" namespace. Then we create "testing.table2"
+ * which creates "table2" and puts it into "testing" as well. Then we make sure that you can't delete a namespace with tables in it, and then we delete the
+ * tables and delete the namespace.
- *
- * @throws Exception
+ */
+ @Test
+ public void testCreateAndDeleteNamespace() throws Exception {
+ String namespace = "testing";
+ String tableName1 = namespace + ".table1";
+ String tableName2 = namespace + ".table2";
+
+ Instance instance = new MockInstance("createdelete");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ c.namespaceOperations().create(namespace);
+ assertTrue(c.namespaceOperations().exists(namespace));
+
+ c.tableOperations().create(tableName1);
+ assertTrue(c.tableOperations().exists(tableName1));
+
+ c.tableOperations().create(tableName2);
+ assertTrue(c.tableOperations().exists(tableName2));
+
+ // deleting
+ try {
+ // can't delete a namespace with tables in it
+ c.namespaceOperations().delete(namespace);
+ fail();
+ } catch (NamespaceNotEmptyException e) {
+ // ignore, supposed to happen
+ }
+ assertTrue(c.namespaceOperations().exists(namespace));
+ assertTrue(c.tableOperations().exists(tableName1));
+ assertTrue(c.tableOperations().exists(tableName2));
+
+ c.tableOperations().delete(tableName2);
+ assertTrue(!c.tableOperations().exists(tableName2));
+ assertTrue(c.namespaceOperations().exists(namespace));
+
+ c.tableOperations().delete(tableName1);
+ assertTrue(!c.tableOperations().exists(tableName1));
+ c.namespaceOperations().delete(namespace);
+ assertTrue(!c.namespaceOperations().exists(namespace));
+ }
+
+ /**
+ * This test creates a namespace, modifies it's properties, and checks to make sure that those properties are applied to its tables. To do something on a
+ * namespace-wide level, use {@link NamespaceOperations}.
+ *
+ * Checks to make sure namespace-level properties are overridden by table-level properties.
+ *
+ * Checks to see if the default namespace's properties work as well.
- *
- * @throws Exception
+ */
+
+ @Test
+ public void testNamespaceProperties() throws Exception {
+ String namespace = "propchange";
+ String tableName1 = namespace + ".table1";
+ String tableName2 = namespace + ".table2";
+
+ String propKey = Property.TABLE_SCAN_MAXMEM.getKey();
+ String propVal = "42K";
+
+ Instance instance = new MockInstance("props");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(tableName1);
+ c.namespaceOperations().setProperty(namespace, propKey, propVal);
+
+ // check the namespace has the property
+ assertTrue(checkNamespaceHasProp(c, namespace, propKey, propVal));
+
+ // check that the table gets it from the namespace
+ assertTrue(checkTableHasProp(c, tableName1, propKey, propVal));
+
+ // test a second table to be sure the first wasn't magical
+ // (also, changed the order, the namespace has the property already)
+ c.tableOperations().create(tableName2);
+ assertTrue(checkTableHasProp(c, tableName2, propKey, propVal));
+
+ // test that table properties override namespace properties
+ String propKey2 = Property.TABLE_FILE_MAX.getKey();
+ String propVal2 = "42";
+ String tablePropVal = "13";
+
+ c.tableOperations().setProperty(tableName2, propKey2, tablePropVal);
+ c.namespaceOperations().setProperty("propchange", propKey2, propVal2);
+
+ assertTrue(checkTableHasProp(c, tableName2, propKey2, tablePropVal));
+
+ // now check that you can change the default namespace's properties
+ propVal = "13K";
+ String tableName = "some_table";
+ c.tableOperations().create(tableName);
+ c.namespaceOperations().setProperty(Namespaces.DEFAULT_NAMESPACE, propKey, propVal);
+
+ assertTrue(checkTableHasProp(c, tableName, propKey, propVal));
+
+ // test the properties server-side by configuring an iterator.
+ // should not show anything with column-family = 'a'
+ String tableName3 = namespace + ".table3";
+ c.tableOperations().create(tableName3);
+
+ IteratorSetting setting = new IteratorSetting(250, "thing", SimpleFilter.class.getName());
+ c.namespaceOperations().attachIterator(namespace, setting);
+
+ BatchWriter bw = c.createBatchWriter(tableName3, new BatchWriterConfig());
+ Mutation m = new Mutation("r");
+ m.put("a", "b", new Value("abcde".getBytes()));
+ bw.addMutation(m);
+ bw.flush();
+ bw.close();
+
+ // Scanner s = c.createScanner(tableName3, Authorizations.EMPTY);
+ // do scanners work correctly in mock?
+ // assertTrue(!s.iterator().hasNext());
+ }
+
+ /**
+ * This test renames and clones two separate table into different namespaces. different namespace.
- *
- * @throws Exception
+ */
+ @Test
+ public void testRenameAndCloneTableToNewNamespace() throws Exception {
+ String namespace1 = "renamed";
+ String namespace2 = "cloned";
+ String tableName = "table";
+ String tableName1 = "renamed.table1";
+ // String tableName2 = "cloned.table2";
+
+ Instance instance = new MockInstance("renameclone");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ c.tableOperations().create(tableName);
+ c.namespaceOperations().create(namespace1);
+ c.namespaceOperations().create(namespace2);
+
+ c.tableOperations().rename(tableName, tableName1);
+
+ assertTrue(c.tableOperations().exists(tableName1));
+ assertTrue(!c.tableOperations().exists(tableName));
+
+ // TODO implement clone in mock
+ /*
+ * c.tableOperations().clone(tableName1, tableName2, false, null, null);
+ *
+ * assertTrue(c.tableOperations().exists(tableName1)); assertTrue(c.tableOperations().exists(tableName2));
+ */
+ return;
+ }
+
+ /**
+ * This test renames a namespace and ensures that its tables are still correct
+ */
+ @Test
+ public void testNamespaceRename() throws Exception {
+ String namespace1 = "n1";
+ String namespace2 = "n2";
+ String table = "t";
+
+ Instance instance = new MockInstance("rename");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ c.namespaceOperations().create(namespace1);
+ c.tableOperations().create(namespace1 + "." + table);
+
+ c.namespaceOperations().rename(namespace1, namespace2);
+
+ assertTrue(!c.namespaceOperations().exists(namespace1));
+ assertTrue(c.namespaceOperations().exists(namespace2));
+ assertTrue(!c.tableOperations().exists(namespace1 + "." + table));
+ assertTrue(c.tableOperations().exists(namespace2 + "." + table));
+ }
+
+ /**
+ * This tests adding iterators to a namespace, listing them, and removing them
+ */
+ @Test
+ public void testNamespaceIterators() throws Exception {
+ Instance instance = new MockInstance("Iterators");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ String namespace = "iterator";
+ String tableName = namespace + ".table";
+ String iter = "thing";
+
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(tableName);
+
+ IteratorSetting setting = new IteratorSetting(250, iter, SimpleFilter.class.getName());
+ HashSet<IteratorScope> scope = new HashSet<IteratorScope>();
+ scope.add(IteratorScope.scan);
+ c.namespaceOperations().attachIterator(namespace, setting, EnumSet.copyOf(scope));
+
+ BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
+ Mutation m = new Mutation("r");
+ m.put("a", "b", new Value("abcde".getBytes(Constants.UTF8)));
+ bw.addMutation(m);
+ bw.flush();
+
+ Scanner s = c.createScanner(tableName, Authorizations.EMPTY);
+ System.out.println(s.iterator().next());
+ // do scanners work correctly in mock?
+ // assertTrue(!s.iterator().hasNext());
+
+ assertTrue(c.namespaceOperations().listIterators(namespace).containsKey(iter));
+ c.namespaceOperations().removeIterator(namespace, iter, EnumSet.copyOf(scope));
+ }
+
+ private boolean checkTableHasProp(Connector c, String t, String propKey, String propVal) throws AccumuloException, TableNotFoundException {
+ for (Entry<String,String> e : c.tableOperations().getProperties(t)) {
+ if (e.getKey().equals(propKey) && e.getValue().equals(propVal)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean checkNamespaceHasProp(Connector c, String n, String propKey, String propVal) throws AccumuloException, NamespaceNotFoundException,
+ AccumuloSecurityException {
+ for (Entry<String,String> e : c.namespaceOperations().getProperties(n)) {
+ if (e.getKey().equals(propKey) && e.getValue().equals(propVal)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public static class SimpleFilter extends Filter {
+ @Override
+ public boolean accept(Key k, Value v) {
+ if (k.getColumnFamily().toString().equals("a"))
+ return false;
+ return true;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
index de6ca21,0000000..d31a788
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
@@@ -1,106 -1,0 +1,103 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.security;
+
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.constraints.Constraint.Environment;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.Mutation;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class VisibilityConstraintTest {
+
+ VisibilityConstraint vc;
+ Environment env;
+ Mutation mutation;
+
+ static final ColumnVisibility good = new ColumnVisibility("good");
+ static final ColumnVisibility bad = new ColumnVisibility("bad");
+
+ static final String D = "don't care";
+
+ static final List<Short> ENOAUTH = Arrays.asList((short) 2);
+
- /**
- * @throws java.lang.Exception
- */
+ @Before
+ public void setUp() throws Exception {
+ vc = new VisibilityConstraint();
+ mutation = new Mutation("r");
+
+ ArrayByteSequence bs = new ArrayByteSequence("good".getBytes(Constants.UTF8));
+
+ AuthorizationContainer ac = createNiceMock(AuthorizationContainer.class);
+ expect(ac.contains(bs)).andReturn(true);
+ replay(ac);
+
+ env = createMock(Environment.class);
+ expect(env.getAuthorizationsContainer()).andReturn(ac);
+ replay(env);
+ }
+
+ @Test
+ public void testNoVisibility() {
+ mutation.put(D, D, D);
+ assertNull("authorized", vc.check(env, mutation));
+ }
+
+ @Test
+ public void testVisibilityNoAuth() {
+ mutation.put(D, D, bad, D);
+ assertEquals("unauthorized", ENOAUTH, vc.check(env, mutation));
+ }
+
+ @Test
+ public void testGoodVisibilityAuth() {
+ mutation.put(D, D, good, D);
+ assertNull("authorized", vc.check(env, mutation));
+ }
+
+ @Test
+ public void testCachedVisibilities() {
+ mutation.put(D, D, good, "v");
+ mutation.put(D, D, good, "v2");
+ assertNull("authorized", vc.check(env, mutation));
+ }
+
+ @Test
+ public void testMixedVisibilities() {
+ mutation.put(D, D, bad, D);
+ mutation.put(D, D, good, D);
+ assertEquals("unauthorized", ENOAUTH, vc.check(env, mutation));
+ }
+
+ @Test
+ @Ignore
+ public void testMalformedVisibility() {
+ // TODO: ACCUMULO-1006 Should test for returning error code 1, but not sure how since ColumnVisibility won't let us construct a bad one in the first place
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
index 4607fdb,e76352a..44947d1
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
@@@ -89,40 -89,29 +89,36 @@@ public class RandomBatchWriter
// create a random value that is a function of the
// row id for verification purposes
byte value[] = createValue(rowid, dataSize);
-
+
m.put(new Text("foo"), new Text("1"), visibility, new Value(value));
-
+
return m;
}
-
+
static class Opts extends ClientOnRequiredTable {
- @Parameter(names="--num", required=true)
+ @Parameter(names = "--num", required = true)
int num = 0;
- @Parameter(names="--min")
+ @Parameter(names = "--min")
long min = 0;
- @Parameter(names="--max")
+ @Parameter(names = "--max")
long max = Long.MAX_VALUE;
- @Parameter(names="--size", required=true, description="size of the value to write")
+ @Parameter(names = "--size", required = true, description = "size of the value to write")
int size = 0;
- @Parameter(names="--vis", converter=VisibilityConverter.class)
+ @Parameter(names = "--vis", converter = VisibilityConverter.class)
ColumnVisibility visiblity = new ColumnVisibility("");
- @Parameter(names="--seed", description="seed for pseudo-random number generator")
+ @Parameter(names = "--seed", description = "seed for pseudo-random number generator")
Long seed = null;
}
-
+
+ public static long abs(long l) {
+ l = Math.abs(l); // abs(Long.MIN_VALUE) == Long.MIN_VALUE...
+ if (l < 0)
+ return 0;
+ return l;
+ }
+
/**
* Writes a specified number of entries to Accumulo using a {@link BatchWriter}.
- *
- * @throws AccumuloException
- * @throws AccumuloSecurityException
- * @throws TableNotFoundException
*/
public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
Opts opts = new Opts();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/NGramIngest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
index 3a211e2,669c76d..30ebd06
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
@@@ -121,9 -120,10 +121,8 @@@ public class TableToFile extends Config
*
* @param args
* instanceName zookeepers username password table columns outputpath
- * @throws Exception
*/
public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(CachedConfiguration.getInstance(), new TableToFile(), args);
- if (res != 0)
- System.exit(res);
+ ToolRunner.run(new Configuration(), new TableToFile(), args);
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
index add4c4c,d98d78b..aa12c71
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
@@@ -59,26 -66,10 +59,23 @@@ public class Query
IntersectingIterator.setColumnFamilies(ii, columns);
bs.addScanIterator(ii);
bs.setRanges(Collections.singleton(new Range()));
+ List<String> result = new ArrayList<String>();
for (Entry<Key,Value> entry : bs) {
- System.out.println(" " + entry.getKey().getColumnQualifier());
+ result.add(entry.getKey().getColumnQualifier().toString());
}
-
+ return result;
+ }
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ BatchScannerOpts bsOpts = new BatchScannerOpts();
+ opts.parseArgs(Query.class.getName(), args, bsOpts);
+ Connector conn = opts.getConnector();
+ BatchScanner bs = conn.createBatchScanner(opts.tableName, opts.auths, bsOpts.scanThreads);
+ bs.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);
+
+ for (String entry : query(bs, opts.terms))
+ System.out.println(" " + entry);
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java
----------------------------------------------------------------------
diff --cc minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java
index f7070dc,0000000..ab84d37
mode 100644,000000..100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java
@@@ -1,264 -1,0 +1,262 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.minicluster;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.ServerSocket;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.regex.Pattern;
+
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.commons.io.FileUtils;
+
+import com.beust.jcommander.IStringConverter;
+import com.beust.jcommander.Parameter;
+import com.google.common.io.Files;
+
+/**
+ * A runner for starting up a {@link MiniAccumuloCluster} from the command line using an optional configuration properties file. An example property file looks
+ * like the following:
+ *
+ * <pre>
+ * rootPassword=secret
+ * instanceName=testInstance
+ * numTServers=1
+ * zooKeeperPort=3191
+ * jdwpEnabled=true
+ * zooKeeperMemory=128M
+ * tserverMemory=256M
+ * masterMemory=128M
+ * defaultMemory=256M
+ * shutdownPort=4446
+ * site.instance.secret=HUSH
+ * </pre>
+ *
+ * All items in the properties file above are optional and a default value will be provided in their absence. Any site configuration properties (typically found
+ * in the accumulo-site.xml file) should be prefixed with "site." in the properties file.
+ *
+ * @since 1.6.0
+ */
+public class MiniAccumuloRunner {
+ private static final String ROOT_PASSWORD_PROP = "rootPassword";
+ private static final String SHUTDOWN_PORT_PROP = "shutdownPort";
+ private static final String DEFAULT_MEMORY_PROP = "defaultMemory";
+ private static final String MASTER_MEMORY_PROP = "masterMemory";
+ private static final String TSERVER_MEMORY_PROP = "tserverMemory";
+ private static final String ZOO_KEEPER_MEMORY_PROP = "zooKeeperMemory";
+ private static final String JDWP_ENABLED_PROP = "jdwpEnabled";
+ private static final String ZOO_KEEPER_PORT_PROP = "zooKeeperPort";
+ private static final String NUM_T_SERVERS_PROP = "numTServers";
+ private static final String DIRECTORY_PROP = "directory";
+ private static final String INSTANCE_NAME_PROP = "instanceName";
+
+ private static void printProperties() {
+ System.out.println("#mini Accumulo cluster runner properties.");
+ System.out.println("#");
+ System.out.println("#uncomment following propeties to use, propeties not set will use default or random value");
+ System.out.println();
+ System.out.println("#" + INSTANCE_NAME_PROP + "=devTest");
+ System.out.println("#" + DIRECTORY_PROP + "=/tmp/mac1");
+ System.out.println("#" + ROOT_PASSWORD_PROP + "=secret");
+ System.out.println("#" + NUM_T_SERVERS_PROP + "=2");
+ System.out.println("#" + ZOO_KEEPER_PORT_PROP + "=40404");
+ System.out.println("#" + SHUTDOWN_PORT_PROP + "=41414");
+ System.out.println("#" + DEFAULT_MEMORY_PROP + "=128M");
+ System.out.println("#" + MASTER_MEMORY_PROP + "=128M");
+ System.out.println("#" + TSERVER_MEMORY_PROP + "=128M");
+ System.out.println("#" + ZOO_KEEPER_MEMORY_PROP + "=128M");
+ System.out.println("#" + JDWP_ENABLED_PROP + "=false");
+
+ System.out.println();
+ System.out.println("# Configuration normally placed in accumulo-site.xml can be added using a site. prefix.");
+ System.out.println("# For example the following line will set tserver.compaction.major.concurrent.max");
+ System.out.println();
+ System.out.println("#site.tserver.compaction.major.concurrent.max=4");
+
+ }
+
+ public static class PropertiesConverter implements IStringConverter<Properties> {
+ @Override
+ public Properties convert(String fileName) {
+ Properties prop = new Properties();
+ InputStream is;
+ try {
+ is = new FileInputStream(fileName);
+ try {
+ prop.load(is);
+ } finally {
+ is.close();
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return prop;
+ }
+ }
+
+ private static final String FORMAT_STRING = " %-21s %s";
+
+ public static class Opts extends Help {
+ @Parameter(names = "-p", required = false, description = "properties file name", converter = PropertiesConverter.class)
+ Properties prop = new Properties();
+
+ @Parameter(names = {"-c", "--printProperties"}, required = false, description = "prints an example propeties file, redirect to file to use")
+ boolean printProps = false;
+ }
+
+ /**
+ * Runs the {@link MiniAccumuloCluster} given a -p argument with a property file. Establishes a shutdown port for asynchronous operation.
+ *
+ * @param args
+ * An optional -p argument can be specified with the path to a valid properties file.
- *
- * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(MiniAccumuloRunner.class.getName(), args);
+
+ if (opts.printProps) {
+ printProperties();
+ System.exit(0);
+ }
+
+ int shutdownPort = 4445;
+
+ final File miniDir;
+
+ if (opts.prop.containsKey(DIRECTORY_PROP))
+ miniDir = new File(opts.prop.getProperty(DIRECTORY_PROP));
+ else
+ miniDir = Files.createTempDir();
+
+ String rootPass = opts.prop.containsKey(ROOT_PASSWORD_PROP) ? opts.prop.getProperty(ROOT_PASSWORD_PROP) : "secret";
+
+ MiniAccumuloConfig config = new MiniAccumuloConfig(miniDir, rootPass);
+
+ if (opts.prop.containsKey(INSTANCE_NAME_PROP))
+ config.setInstanceName(opts.prop.getProperty(INSTANCE_NAME_PROP));
+ if (opts.prop.containsKey(NUM_T_SERVERS_PROP))
+ config.setNumTservers(Integer.parseInt(opts.prop.getProperty(NUM_T_SERVERS_PROP)));
+ if (opts.prop.containsKey(ZOO_KEEPER_PORT_PROP))
+ config.setZooKeeperPort(Integer.parseInt(opts.prop.getProperty(ZOO_KEEPER_PORT_PROP)));
+ if (opts.prop.containsKey(JDWP_ENABLED_PROP))
+ config.setJDWPEnabled(Boolean.parseBoolean(opts.prop.getProperty(JDWP_ENABLED_PROP)));
+ if (opts.prop.containsKey(ZOO_KEEPER_MEMORY_PROP))
+ setMemoryOnConfig(config, opts.prop.getProperty(ZOO_KEEPER_MEMORY_PROP), ServerType.ZOOKEEPER);
+ if (opts.prop.containsKey(TSERVER_MEMORY_PROP))
+ setMemoryOnConfig(config, opts.prop.getProperty(TSERVER_MEMORY_PROP), ServerType.TABLET_SERVER);
+ if (opts.prop.containsKey(MASTER_MEMORY_PROP))
+ setMemoryOnConfig(config, opts.prop.getProperty(MASTER_MEMORY_PROP), ServerType.MASTER);
+ if (opts.prop.containsKey(DEFAULT_MEMORY_PROP))
+ setMemoryOnConfig(config, opts.prop.getProperty(DEFAULT_MEMORY_PROP));
+ if (opts.prop.containsKey(SHUTDOWN_PORT_PROP))
+ shutdownPort = Integer.parseInt(opts.prop.getProperty(SHUTDOWN_PORT_PROP));
+
+ Map<String,String> siteConfig = new HashMap<String,String>();
+ for (Map.Entry<Object,Object> entry : opts.prop.entrySet()) {
+ String key = (String) entry.getKey();
+ if (key.startsWith("site."))
+ siteConfig.put(key.replaceFirst("site.", ""), (String) entry.getValue());
+ }
+
+ config.setSiteConfig(siteConfig);
+
+ final MiniAccumuloCluster accumulo = new MiniAccumuloCluster(config);
+
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ try {
+ accumulo.stop();
+ FileUtils.deleteDirectory(miniDir);
+ System.out.println("\nShut down gracefully on " + new Date());
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+
+ accumulo.start();
+
+ printInfo(accumulo, shutdownPort);
+
+ // start a socket on the shutdown port and block- anything connected to this port will activate the shutdown
+ ServerSocket shutdownServer = new ServerSocket(shutdownPort);
+ shutdownServer.accept();
+
+ System.exit(0);
+ }
+
+ private static boolean validateMemoryString(String memoryString) {
+ String unitsRegex = "[";
+ MemoryUnit[] units = MemoryUnit.values();
+ for (int i = 0; i < units.length; i++) {
+ unitsRegex += units[i].suffix();
+ if (i < units.length - 1)
+ unitsRegex += "|";
+ }
+ unitsRegex += "]";
+ Pattern p = Pattern.compile("\\d+" + unitsRegex);
+ return p.matcher(memoryString).matches();
+ }
+
+ private static void setMemoryOnConfig(MiniAccumuloConfig config, String memoryString) {
+ setMemoryOnConfig(config, memoryString, null);
+ }
+
+ private static void setMemoryOnConfig(MiniAccumuloConfig config, String memoryString, ServerType serverType) {
+ if (!validateMemoryString(memoryString))
+ throw new IllegalArgumentException(memoryString + " is not a valid memory string");
+
+ long memSize = Long.parseLong(memoryString.substring(0, memoryString.length() - 1));
+ MemoryUnit memUnit = MemoryUnit.fromSuffix(memoryString.substring(memoryString.length() - 1));
+
+ if (serverType != null)
+ config.setMemory(serverType, memSize, memUnit);
+ else
+ config.setDefaultMemory(memSize, memUnit);
+ }
+
+ private static void printInfo(MiniAccumuloCluster accumulo, int shutdownPort) {
+ System.out.println("Mini Accumulo Cluster\n");
+ System.out.println(String.format(FORMAT_STRING, "Directory:", accumulo.getConfig().getDir().getAbsoluteFile()));
+ System.out.println(String.format(FORMAT_STRING, "Logs:", accumulo.getConfig().getImpl().getLogDir().getAbsoluteFile()));
+ System.out.println(String.format(FORMAT_STRING, "Instance Name:", accumulo.getConfig().getInstanceName()));
+ System.out.println(String.format(FORMAT_STRING, "Root Password:", accumulo.getConfig().getRootPassword()));
+ System.out.println(String.format(FORMAT_STRING, "ZooKeeper:", accumulo.getZooKeepers()));
+
+ for (Pair<ServerType,Integer> pair : accumulo.getDebugPorts()) {
+ System.out.println(String.format(FORMAT_STRING, pair.getFirst().prettyPrint() + " JDWP Host:", "localhost:" + pair.getSecond()));
+ }
+
+ System.out.println(String.format(FORMAT_STRING, "Shutdown Port:", shutdownPort));
+
+ System.out.println();
+ System.out.println(" To connect with shell, use the following command : ");
+ System.out.println(" accumulo shell -zh " + accumulo.getZooKeepers() + " -zi " + accumulo.getConfig().getInstanceName() + " -u root ");
+
+ System.out.println("\n\nSuccessfully started on " + new Date());
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java
index 442294f,0000000..05806ca
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java
@@@ -1,30 -1,0 +1,27 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.conf;
+
+import org.apache.accumulo.server.client.HdfsZooInstance;
+
+public class ConfigSanityCheck {
+
- /**
- * @param args
- */
+ public static void main(String[] args) {
+ new ServerConfiguration(HdfsZooInstance.getInstance()).getConfiguration();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
index fd76ce2,0000000..5bd1632
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
@@@ -1,151 -1,0 +1,149 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.balancer;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletMigration;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransportException;
+
+public abstract class TabletBalancer {
+
+ private static final Logger log = Logger.getLogger(TabletBalancer.class);
+
+ protected ServerConfiguration configuration;
+
+ /**
+ * Initialize the TabletBalancer. This gives the balancer the opportunity to read the configuration.
+ */
+ public void init(ServerConfiguration conf) {
+ configuration = conf;
+ }
+
+ /**
+ * Assign tablets to tablet servers. This method is called whenever the master finds tablets that are unassigned.
+ *
+ * @param current
+ * The current table-summary state of all the online tablet servers. Read-only. The TabletServerStatus for each server may be null if the tablet
+ * server has not yet responded to a recent request for status.
+ * @param unassigned
+ * A map from unassigned tablet to the last known tablet server. Read-only.
+ * @param assignments
+ * A map from tablet to assigned server. Write-only.
+ */
+ abstract public void getAssignments(SortedMap<TServerInstance,TabletServerStatus> current, Map<KeyExtent,TServerInstance> unassigned,
+ Map<KeyExtent,TServerInstance> assignments);
+
+ /**
+ * Ask the balancer if any migrations are necessary.
+ *
+ * @param current
+ * The current table-summary state of all the online tablet servers. Read-only.
+ * @param migrations
+ * the current set of migrations. Read-only.
+ * @param migrationsOut
+ * new migrations to perform; should not contain tablets in the current set of migrations. Write-only.
+ * @return the time, in milliseconds, to wait before re-balancing.
+ *
+ * This method will not be called when there are unassigned tablets.
+ */
+ public abstract long balance(SortedMap<TServerInstance,TabletServerStatus> current, Set<KeyExtent> migrations, List<TabletMigration> migrationsOut);
+
+ /**
+ * Fetch the tablets for the given table by asking the tablet server. Useful if your balance strategy needs details at the tablet level to decide what tablets
+ * to move.
+ *
+ * @param tserver
+ * The tablet server to ask.
+ * @param tableId
+ * The table id
+ * @return a list of tablet statistics
+ * @throws ThriftSecurityException
+ * tablet server disapproves of your internal System password.
+ * @throws TException
+ * any other problem
+ */
+ public List<TabletStats> getOnlineTabletsForTable(TServerInstance tserver, String tableId) throws ThriftSecurityException, TException {
+ log.debug("Scanning tablet server " + tserver + " for table " + tableId);
+ Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), tserver.getLocation(), configuration.getConfiguration());
+ try {
+ List<TabletStats> onlineTabletsForTable = client.getTabletStats(Tracer.traceInfo(), SystemCredentials.get().toThrift(configuration.getInstance()),
+ tableId);
+ return onlineTabletsForTable;
+ } catch (TTransportException e) {
+ log.error("Unable to connect to " + tserver + ": " + e);
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ return null;
+ }
+
+ /**
+ * Utility to ensure that the migrations from balance() are consistent:
+ * <ul>
+ * <li>Tablet objects are not null
+ * <li>Source and destination tablet servers are not null and current
+ * </ul>
+ *
- * @param current
- * @param migrations
+ * @return A list of TabletMigration object that passed sanity checks.
+ */
+ public static List<TabletMigration> checkMigrationSanity(Set<TServerInstance> current, List<TabletMigration> migrations) {
+ List<TabletMigration> result = new ArrayList<TabletMigration>(migrations.size());
+ for (TabletMigration m : migrations) {
+ if (m.tablet == null) {
+ log.warn("Balancer gave back a null tablet " + m);
+ continue;
+ }
+ if (m.newServer == null) {
+ log.warn("Balancer did not set the destination " + m);
+ continue;
+ }
+ if (m.oldServer == null) {
+ log.warn("Balancer did not set the source " + m);
+ continue;
+ }
+ if (!current.contains(m.oldServer)) {
+ log.warn("Balancer wants to move a tablet from a server that is not current: " + m);
+ continue;
+ }
+ if (!current.contains(m.newServer)) {
+ log.warn("Balancer wants to move a tablet to a server that is not current: " + m);
+ continue;
+ }
+ result.add(m);
+ }
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
index e898109,0000000..7c75454
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
@@@ -1,91 -1,0 +1,84 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.state;
+
+import java.util.Collection;
+import java.util.Collections;
+
+/**
+ * Interface for storing information about tablet assignments. There are three implementations:
+ *
+ * ZooTabletStateStore: information about the root tablet is stored in ZooKeeper MetaDataStateStore: information about the other tablets are stored in the
+ * metadata table
+ *
+ */
+public abstract class TabletStateStore implements Iterable<TabletLocationState> {
+
+ /**
+ * Identifying name for this tablet state store.
+ */
+ abstract public String name();
+
+ /**
+ * Scan the information about the tablets covered by this store
+ */
+ @Override
+ abstract public ClosableIterator<TabletLocationState> iterator();
+
+ /**
+ * Store the assigned locations in the data store.
- *
- * @param assignments
- * @throws DistributedStoreException
+ */
+ abstract public void setFutureLocations(Collection<Assignment> assignments) throws DistributedStoreException;
+
+ /**
+ * Tablet servers will update the data store with the location when they bring the tablet online
- *
- * @param assignments
- * @throws DistributedStoreException
+ */
+ abstract public void setLocations(Collection<Assignment> assignments) throws DistributedStoreException;
+
+ /**
+ * Mark the tablets as having no known or future location.
+ *
+ * @param tablets
+ * the tablets' current information
- * @throws DistributedStoreException
+ */
+ abstract public void unassign(Collection<TabletLocationState> tablets) throws DistributedStoreException;
+
+ public static void unassign(TabletLocationState tls) throws DistributedStoreException {
+ TabletStateStore store;
+ if (tls.extent.isRootTablet()) {
+ store = new ZooTabletStateStore();
+ } else if (tls.extent.isMeta()) {
+ store = new RootTabletStateStore();
+ } else {
+ store = new MetaDataStateStore();
+ }
+ store.unassign(Collections.singletonList(tls));
+ }
+
+ public static void setLocation(Assignment assignment) throws DistributedStoreException {
+ TabletStateStore store;
+ if (assignment.tablet.isRootTablet()) {
+ store = new ZooTabletStateStore();
+ } else if (assignment.tablet.isMeta()) {
+ store = new RootTabletStateStore();
+ } else {
+ store = new MetaDataStateStore();
+ }
+ store.setLocations(Collections.singletonList(assignment));
+ }
+
+}
[33/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
index 156fa3a,0000000..2a1372c
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
@@@ -1,242 -1,0 +1,209 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.admin;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.ClientExec;
+import org.apache.accumulo.core.client.impl.ClientExecReturn;
+import org.apache.accumulo.core.client.impl.MasterClient;
+import org.apache.accumulo.core.client.impl.ServerClient;
+import org.apache.accumulo.core.client.impl.thrift.ClientService;
+import org.apache.accumulo.core.client.impl.thrift.ConfigurationType;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+
+/**
+ * Provides a class for administering the accumulo instance
+ */
+public class InstanceOperationsImpl implements InstanceOperations {
+ private Instance instance;
+ private TCredentials credentials;
+
+ /**
+ * @param instance
+ * the connection information for this instance
+ * @param credentials
+ * the Credential, containing principal and Authentication Token
+ */
+ public InstanceOperationsImpl(Instance instance, TCredentials credentials) {
+ ArgumentChecker.notNull(instance, credentials);
+ this.instance = instance;
+ this.credentials = credentials;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#setProperty(java.lang.String, java.lang.String)
- */
+ @Override
+ public void setProperty(final String property, final String value) throws AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(property, value);
+ MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() {
+ @Override
+ public void execute(MasterClientService.Client client) throws Exception {
+ client.setSystemProperty(Tracer.traceInfo(), credentials, property, value);
+ }
+ });
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#removeProperty(java.lang.String)
- */
+ @Override
+ public void removeProperty(final String property) throws AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(property);
+ MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() {
+ @Override
+ public void execute(MasterClientService.Client client) throws Exception {
+ client.removeSystemProperty(Tracer.traceInfo(), credentials, property);
+ }
+ });
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getSystemConfiguration()
- */
+ @Override
+ public Map<String,String> getSystemConfiguration() throws AccumuloException, AccumuloSecurityException {
+ return ServerClient.execute(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() {
+ @Override
+ public Map<String,String> execute(ClientService.Client client) throws Exception {
+ return client.getConfiguration(Tracer.traceInfo(), credentials, ConfigurationType.CURRENT);
+ }
+ });
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getSiteConfiguration()
- */
+ @Override
+ public Map<String,String> getSiteConfiguration() throws AccumuloException, AccumuloSecurityException {
+ return ServerClient.execute(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() {
+ @Override
+ public Map<String,String> execute(ClientService.Client client) throws Exception {
+ return client.getConfiguration(Tracer.traceInfo(), credentials, ConfigurationType.SITE);
+ }
+ });
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getTabletServers()
- */
-
+ @Override
+ public List<String> getTabletServers() {
+ ZooCache cache = ZooCache.getInstance(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
+ String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
+ List<String> results = new ArrayList<String>();
+ for (String candidate : cache.getChildren(path)) {
+ List<String> children = cache.getChildren(path + "/" + candidate);
+ if (children != null && children.size() > 0) {
+ List<String> copy = new ArrayList<String>(children);
+ Collections.sort(copy);
+ byte[] data = cache.get(path + "/" + candidate + "/" + copy.get(0));
+ if (data != null && !"master".equals(new String(data, Constants.UTF8))) {
+ results.add(candidate);
+ }
+ }
+ }
+ return results;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getActiveScans(java.lang.String)
- */
-
+ @Override
+ public List<ActiveScan> getActiveScans(String tserver) throws AccumuloException, AccumuloSecurityException {
+ Client client = null;
+ try {
+ client = ThriftUtil.getTServerClient(tserver, instance.getConfiguration());
+
+ List<ActiveScan> as = new ArrayList<ActiveScan>();
+ for (org.apache.accumulo.core.tabletserver.thrift.ActiveScan activeScan : client.getActiveScans(Tracer.traceInfo(), credentials)) {
+ try {
+ as.add(new ActiveScan(instance, activeScan));
+ } catch (TableNotFoundException e) {
+ throw new AccumuloException(e);
+ }
+ }
+ return as;
+ } catch (TTransportException e) {
+ throw new AccumuloException(e);
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (TException e) {
+ throw new AccumuloException(e);
+ } finally {
+ if (client != null)
+ ThriftUtil.returnClient(client);
+ }
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#testClassLoad(java.lang.String, java.lang.String)
- */
+ @Override
+ public boolean testClassLoad(final String className, final String asTypeName) throws AccumuloException, AccumuloSecurityException {
+ return ServerClient.execute(instance, new ClientExecReturn<Boolean,ClientService.Client>() {
+ @Override
+ public Boolean execute(ClientService.Client client) throws Exception {
+ return client.checkClass(Tracer.traceInfo(), credentials, className, asTypeName);
+ }
+ });
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getActiveCompactions(java.lang.String)
- */
+ @Override
+ public List<ActiveCompaction> getActiveCompactions(String tserver) throws AccumuloException, AccumuloSecurityException {
+ Client client = null;
+ try {
+ client = ThriftUtil.getTServerClient(tserver, instance.getConfiguration());
+
+ List<ActiveCompaction> as = new ArrayList<ActiveCompaction>();
+ for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction activeCompaction : client.getActiveCompactions(Tracer.traceInfo(), credentials)) {
+ as.add(new ActiveCompaction(instance, activeCompaction));
+ }
+ return as;
+ } catch (TTransportException e) {
+ throw new AccumuloException(e);
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (TException e) {
+ throw new AccumuloException(e);
+ } finally {
+ if (client != null)
+ ThriftUtil.returnClient(client);
+ }
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#ping(java.lang.String)
- */
+ @Override
+ public void ping(String tserver) throws AccumuloException {
+ TTransport transport = null;
+ try {
+ transport = ThriftUtil.createTransport(tserver, instance.getConfiguration().getPort(Property.TSERV_CLIENTPORT), instance.getConfiguration());
+ TabletClientService.Client client = ThriftUtil.createClient(new TabletClientService.Client.Factory(), transport);
+ client.getTabletServerStatus(Tracer.traceInfo(), credentials);
+ } catch (TTransportException e) {
+ throw new AccumuloException(e);
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloException(e);
+ } catch (TException e) {
+ throw new AccumuloException(e);
+ } finally {
+ if (transport != null) {
+ transport.close();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
index 2521c96,0000000..0823656
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
@@@ -1,719 -1,0 +1,687 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.admin;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedSet;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Provides a class for administering tables
+ *
+ */
+
+public interface TableOperations {
+
+ /**
+ * Retrieve a list of tables in Accumulo.
+ *
+ * @return List of tables in accumulo
+ */
+ public SortedSet<String> list();
+
+ /**
+ * A method to check if a table exists in Accumulo.
+ *
+ * @param tableName
+ * the name of the table
+ * @return true if the table exists
+ */
+ public boolean exists(String tableName);
+
+ /**
+ * Create a table with no special configuration
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableExistsException
+ * if the table already exists
+ */
+ public void create(String tableName) throws AccumuloException, AccumuloSecurityException, TableExistsException;
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param limitVersion
+ * Enables/disables the versioning iterator, which will limit the number of Key versions kept.
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableExistsException
+ * if the table already exists
+ */
+ public void create(String tableName, boolean limitVersion) throws AccumuloException, AccumuloSecurityException, TableExistsException;
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param versioningIter
+ * Enables/disables the versioning iterator, which will limit the number of Key versions kept.
+ * @param timeType
+ * specifies logical or real-time based time recording for entries in the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableExistsException
+ * if the table already exists
+ */
+ public void create(String tableName, boolean versioningIter, TimeType timeType) throws AccumuloException, AccumuloSecurityException, TableExistsException;
+
+ /**
+ * Imports a table exported via exportTable and copied via hadoop distcp.
+ *
+ * @param tableName
+ * Name of a table to create and import into.
+ * @param importDir
+ * Directory that contains the files copied by distcp from exportTable
- * @throws TableExistsException
- * @throws AccumuloException
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public void importTable(String tableName, String importDir) throws TableExistsException, AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Exports a table. The tables data is not exported, just table metadata and a list of files to distcp. The table being exported must be offline and stay
+ * offline for the duration of distcp. To avoid losing access to a table it can be cloned and the clone taken offline for export.
+ *
+ * <p>
+ * See docs/examples/README.export
+ *
+ * @param tableName
+ * Name of the table to export.
+ * @param exportDir
+ * An empty directory in HDFS where files containing table metadata and list of files to distcp will be placed.
- * @throws TableNotFoundException
- * @throws AccumuloException
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public void exportTable(String tableName, String exportDir) throws TableNotFoundException, AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Ensures that tablets are split along a set of keys.
+ * <p>
+ * Note that while the documentation for Text specifies that its bytestream should be UTF-8, the encoding is not enforced by operations that work with byte arrays.
+ * <p>
+ * For example, you can create 256 evenly-sliced splits via the following code sample even though the given byte sequences are not valid UTF-8.
+ * <pre>
+ * {@code
+ * TableOperations tableOps = connector.tableOperations();
+ * TreeSet<Text> splits = new TreeSet<Text>();
+ * for (int i = 0; i < 256; i++) {
+ * byte[] bytes = { (byte) i };
+ * splits.add(new Text(bytes));
+ * }
+ * tableOps.addSplits(TABLE_NAME, splits);
+ * }
+ * </pre>
+ *
+ * @param tableName
+ * the name of the table
+ * @param partitionKeys
+ * a sorted set of row key values to pre-split the table on
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ public void addSplits(String tableName, SortedSet<Text> partitionKeys) throws TableNotFoundException, AccumuloException, AccumuloSecurityException;
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @return the split points (end-row names) for the table's current split profile
+ * @throws TableNotFoundException
+ * if the table does not exist
+ * @deprecated since 1.5.0; use {@link #listSplits(String)} instead.
+ */
+ @Deprecated
+ public Collection<Text> getSplits(String tableName) throws TableNotFoundException;
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @return the split points (end-row names) for the table's current split profile
+ * @throws TableNotFoundException
+ * if the table does not exist
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @since 1.5.0
+ */
+ public Collection<Text> listSplits(String tableName) throws TableNotFoundException, AccumuloSecurityException, AccumuloException;
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param maxSplits
+ * specifies the maximum number of splits to return
+ * @return the split points (end-row names) for the table's current split profile, grouped into fewer splits so as not to exceed maxSplits
- * @throws TableNotFoundException
+ * @deprecated since 1.5.0; use {@link #listSplits(String, int)} instead.
+ */
+ @Deprecated
+ public Collection<Text> getSplits(String tableName, int maxSplits) throws TableNotFoundException;
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param maxSplits
+ * specifies the maximum number of splits to return
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @return the split points (end-row names) for the table's current split profile, grouped into fewer splits so as not to exceed maxSplits
- * @throws TableNotFoundException
+ * @since 1.5.0
+ */
+ public Collection<Text> listSplits(String tableName, int maxSplits) throws TableNotFoundException, AccumuloSecurityException, AccumuloException;
+
+ /**
+ * Finds the max row within a given range. To find the max row in a table, pass null for start and end row.
+ *
- * @param tableName
+ * @param auths
+ * find the max row that can seen with these auths
+ * @param startRow
+ * row to start looking at, null means -Infinity
+ * @param startInclusive
+ * determines if the start row is included
+ * @param endRow
+ * row to stop looking at, null means Infinity
+ * @param endInclusive
+ * determines if the end row is included
+ *
+ * @return The max row in the range, or null if there is no visible data in the range.
- *
- * @throws AccumuloSecurityException
- * @throws AccumuloException
- * @throws TableNotFoundException
+ */
+ public Text getMaxRow(String tableName, Authorizations auths, Text startRow, boolean startInclusive, Text endRow, boolean endInclusive)
+ throws TableNotFoundException, AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Merge tablets between (start, end]
+ *
+ * @param tableName
+ * the table to merge
+ * @param start
+ * first tablet to be merged contains the row after this row, null means the first tablet
+ * @param end
+ * last tablet to be merged contains this row, null means the last tablet
+ */
+ public void merge(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
+ /**
+ * Delete rows between (start, end]
+ *
+ * @param tableName
+ * the table to merge
+ * @param start
+ * delete rows after this, null means the first row of the table
+ * @param end
+ * last row to be deleted, inclusive, null means the last row of the table
+ */
+ public void deleteRows(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
+ /**
+ * Starts a full major compaction of the tablets in the range (start, end]. The compaction is preformed even for tablets that have only one file.
+ *
+ * @param tableName
+ * the table to compact
+ * @param start
+ * first tablet to be compacted contains the row after this row, null means the first tablet in table
+ * @param end
+ * last tablet to be merged contains this row, null means the last tablet in table
+ * @param flush
+ * when true, table memory is flushed before compaction starts
+ * @param wait
+ * when true, the call will not return until compactions are finished
+ */
+ public void compact(String tableName, Text start, Text end, boolean flush, boolean wait) throws AccumuloSecurityException, TableNotFoundException,
+ AccumuloException;
+
+ /**
+ * Starts a full major compaction of the tablets in the range (start, end]. The compaction is preformed even for tablets that have only one file.
+ *
+ * @param tableName
+ * the table to compact
+ * @param start
+ * first tablet to be compacted contains the row after this row, null means the first tablet in table
+ * @param end
+ * last tablet to be merged contains this row, null means the last tablet in table
+ * @param iterators
+ * A set of iterators that will be applied to each tablet compacted
+ * @param flush
+ * when true, table memory is flushed before compaction starts
+ * @param wait
+ * when true, the call will not return until compactions are finished
+ * @since 1.5.0
+ */
+ public void compact(String tableName, Text start, Text end, List<IteratorSetting> iterators, boolean flush, boolean wait) throws AccumuloSecurityException,
+ TableNotFoundException, AccumuloException;
+
+ /**
+ * Cancels a user initiated major compaction of a table initiated with {@link #compact(String, Text, Text, boolean, boolean)} or
+ * {@link #compact(String, Text, Text, List, boolean, boolean)}. Compactions of tablets that are currently running may finish, but new compactions of tablets
+ * will not start.
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws TableNotFoundException
+ * if the table does not exist
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @since 1.5.0
+ */
+ public void cancelCompaction(String tableName) throws AccumuloSecurityException, TableNotFoundException, AccumuloException;
+
+ /**
+ * Delete a table
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ public void delete(String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
+ /**
+ * Clone a table from an existing table. The cloned table will have the same data as the source table it was created from. After cloning, the two tables can
+ * mutate independently. Initially the cloned table should not use any extra space, however as the source table and cloned table major compact extra space
+ * will be used by the clone.
+ *
+ * Initially the cloned table is only readable and writable by the user who created it.
+ *
+ * @param srcTableName
+ * the table to clone
+ * @param newTableName
+ * the name of the clone
+ * @param flush
+ * determines if memory is flushed in the source table before cloning.
+ * @param propertiesToSet
+ * the sources tables properties are copied, this allows overriding of those properties
+ * @param propertiesToExclude
+ * do not copy these properties from the source table, just revert to system defaults
+ */
+
+ public void clone(String srcTableName, String newTableName, boolean flush, Map<String,String> propertiesToSet, Set<String> propertiesToExclude)
+ throws AccumuloException, AccumuloSecurityException, TableNotFoundException, TableExistsException;
+
+ /**
+ * Rename a table
+ *
+ * @param oldTableName
+ * the old table name
+ * @param newTableName
+ * the new table name
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the old table name does not exist
+ * @throws TableExistsException
+ * if the new table name already exists
+ */
+ public void rename(String oldTableName, String newTableName) throws AccumuloSecurityException, TableNotFoundException, AccumuloException,
+ TableExistsException;
+
+ /**
+ * Initiate a flush of a table's data that is in memory
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ *
+ * @deprecated As of release 1.4, replaced by {@link #flush(String, Text, Text, boolean)}
+ */
+ @Deprecated
+ public void flush(String tableName) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Flush a table's data that is currently in memory.
+ *
+ * @param tableName
+ * the name of the table
+ * @param wait
+ * if true the call will not return until all data present in memory when the call was is flushed if false will initiate a flush of data in memory,
+ * but will not wait for it to complete
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
- * @throws TableNotFoundException
+ */
+ public void flush(String tableName, Text start, Text end, boolean wait) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
+ /**
+ * Sets a property on a table. Note that it may take a short period of time (a second) to propagate the change everywhere.
+ *
+ * @param tableName
+ * the name of the table
+ * @param property
+ * the name of a per-table property
+ * @param value
+ * the value to set a per-table property to
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ public void setProperty(String tableName, String property, String value) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Removes a property from a table. Note that it may take a short period of time (a second) to propagate the change everywhere.
+ *
+ * @param tableName
+ * the name of the table
+ * @param property
+ * the name of a per-table property
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ public void removeProperty(String tableName, String property) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Gets properties of a table. Note that recently changed properties may not be available immediately.
+ *
+ * @param tableName
+ * the name of the table
+ * @return all properties visible by this table (system and per-table properties). Note that recently changed
+ * properties may not be visible immediately.
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ public Iterable<Entry<String,String>> getProperties(String tableName) throws AccumuloException, TableNotFoundException;
+
+ /**
+ * Sets a table's locality groups. A table's locality groups can be changed at any time.
+ *
+ * @param tableName
+ * the name of the table
+ * @param groups
+ * mapping of locality group names to column families in the locality group
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ public void setLocalityGroups(String tableName, Map<String,Set<Text>> groups) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
+ /**
+ *
+ * Gets the locality groups currently set for a table.
+ *
+ * @param tableName
+ * the name of the table
+ * @return mapping of locality group names to column families in the locality group
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ public Map<String,Set<Text>> getLocalityGroups(String tableName) throws AccumuloException, TableNotFoundException;
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param range
+ * a range to split
+ * @param maxSplits
+ * the maximum number of splits
+ * @return the range, split into smaller ranges that fall on boundaries of the table's split points as evenly as possible
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ public Set<Range> splitRangeByTablets(String tableName, Range range, int maxSplits) throws AccumuloException, AccumuloSecurityException,
+ TableNotFoundException;
+
+ /**
+ * Bulk import all the files in a directory into a table.
+ *
+ * @param tableName
+ * the name of the table
+ * @param dir
+ * the HDFS directory to find files for importing
+ * @param failureDir
+ * the HDFS directory to place files that failed to be imported, must exist and be empty
+ * @param setTime
+ * override the time values in the input files, and use the current time for all mutations
+ * @throws IOException
+ * when there is an error reading/writing to HDFS
+ * @throws AccumuloException
+ * when there is a general accumulo error
+ * @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
+ * @throws TableNotFoundException
+ * when the table no longer exists
+ *
+ */
+ public void importDirectory(String tableName, String dir, String failureDir, boolean setTime) throws TableNotFoundException, IOException, AccumuloException,
+ AccumuloSecurityException;
+
+ /**
+ *
+ * @param tableName
+ * the table to take offline
+ * @throws AccumuloException
+ * when there is a general accumulo error
+ * @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
- * @throws TableNotFoundException
+ */
+ public void offline(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
+
+ /**
+ *
+ * @param tableName
+ * the table to take online
+ * @throws AccumuloException
+ * when there is a general accumulo error
+ * @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
- * @throws TableNotFoundException
+ */
+ public void online(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
+
+ /**
+ * Clears the tablet locator cache for a specified table
+ *
+ * @param tableName
+ * the name of the table
+ * @throws TableNotFoundException
+ * if table does not exist
+ */
+ public void clearLocatorCache(String tableName) throws TableNotFoundException;
+
+ /**
+ * Get a mapping of table name to internal table id.
+ *
+ * @return the map from table name to internal table id
+ */
+ public Map<String,String> tableIdMap();
+
+ /**
+ * Add an iterator to a table on all scopes.
+ *
+ * @param tableName
+ * the name of the table
+ * @param setting
+ * object specifying the properties of the iterator
+ * @throws AccumuloSecurityException
+ * thrown if the user does not have the ability to set properties on the table
- * @throws AccumuloException
+ * @throws TableNotFoundException
+ * throw if the table no longer exists
+ * @throws IllegalArgumentException
+ * if the setting conflicts with any existing iterators
+ */
+ public void attachIterator(String tableName, IteratorSetting setting) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
+
+ /**
+ * Add an iterator to a table on the given scopes.
+ *
+ * @param tableName
+ * the name of the table
+ * @param setting
+ * object specifying the properties of the iterator
+ * @throws AccumuloSecurityException
+ * thrown if the user does not have the ability to set properties on the table
- * @throws AccumuloException
+ * @throws TableNotFoundException
+ * throw if the table no longer exists
+ * @throws IllegalArgumentException
+ * if the setting conflicts with any existing iterators
+ */
+ public void attachIterator(String tableName, IteratorSetting setting, EnumSet<IteratorScope> scopes) throws AccumuloSecurityException, AccumuloException,
+ TableNotFoundException;
+
+ /**
+ * Remove an iterator from a table by name.
+ *
+ * @param tableName
+ * the name of the table
+ * @param name
+ * the name of the iterator
+ * @param scopes
+ * the scopes of the iterator
+ * @throws AccumuloSecurityException
+ * thrown if the user does not have the ability to set properties on the table
- * @throws AccumuloException
+ * @throws TableNotFoundException
+ * throw if the table no longer exists
+ */
+ public void removeIterator(String tableName, String name, EnumSet<IteratorScope> scopes) throws AccumuloSecurityException, AccumuloException,
+ TableNotFoundException;
+
+ /**
+ * Get the settings for an iterator.
+ *
+ * @param tableName
+ * the name of the table
+ * @param name
+ * the name of the iterator
+ * @param scope
+ * the scope of the iterator
+ * @return the settings for this iterator
+ * @throws AccumuloSecurityException
+ * thrown if the user does not have the ability to set properties on the table
- * @throws AccumuloException
+ * @throws TableNotFoundException
+ * throw if the table no longer exists
+ */
+ public IteratorSetting getIteratorSetting(String tableName, String name, IteratorScope scope) throws AccumuloSecurityException, AccumuloException,
+ TableNotFoundException;
+
+ /**
+ * Get a list of iterators for this table.
+ *
+ * @param tableName
+ * the name of the table
+ * @return a set of iterator names
- * @throws AccumuloSecurityException
- * @throws AccumuloException
- * @throws TableNotFoundException
+ */
+ public Map<String,EnumSet<IteratorScope>> listIterators(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
+
+ /**
+ * Check whether a given iterator configuration conflicts with existing configuration; in particular, determine if the name or priority are already in use for
+ * the specified scopes.
+ *
+ * @param tableName
+ * the name of the table
+ * @param setting
+ * object specifying the properties of the iterator
- * @throws AccumuloException
- * @throws TableNotFoundException
+ * @throws IllegalStateException
+ * if the setting conflicts with any existing iterators
+ */
+ public void checkIteratorConflicts(String tableName, IteratorSetting setting, EnumSet<IteratorScope> scopes) throws AccumuloException, TableNotFoundException;
+
+ /**
+ * Add a new constraint to a table.
+ *
+ * @param tableName
+ * the name of the table
+ * @param constraintClassName
+ * the full name of the constraint class
+ * @return the unique number assigned to the constraint
+ * @throws AccumuloException
+ * thrown if the constraint has already been added to the table or if there are errors in the configuration of existing constraints
+ * @throws AccumuloSecurityException
+ * thrown if the user doesn't have permission to add the constraint
- * @throws TableNotFoundException
+ * @since 1.5.0
+ */
+ public int addConstraint(String tableName, String constraintClassName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
+ /**
+ * Remove a constraint from a table.
+ *
+ * @param tableName
+ * the name of the table
+ * @param number
+ * the unique number assigned to the constraint
- * @throws AccumuloException
+ * @throws AccumuloSecurityException
+ * thrown if the user doesn't have permission to remove the constraint
+ * @since 1.5.0
+ */
+ public void removeConstraint(String tableName, int number) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * List constraints on a table with their assigned numbers.
+ *
+ * @param tableName
+ * the name of the table
+ * @return a map from constraint class name to assigned number
+ * @throws AccumuloException
+ * thrown if there are errors in the configuration of existing constraints
- * @throws TableNotFoundException
+ * @since 1.5.0
+ */
+ public Map<String,Integer> listConstraints(String tableName) throws AccumuloException, TableNotFoundException;
+
+ /**
+ * Test to see if the instance can load the given class as the given type. This check uses the table classpath if it is set.
+ *
- * @param className
- * @param asTypeName
+ * @return true if the instance can load the given class as the given type, false otherwise
- * @throws AccumuloException
- *
+ *
+ * @since 1.5.0
+ */
+ public boolean testClassLoad(String tableName, final String className, final String asTypeName) throws AccumuloException, AccumuloSecurityException,
+ TableNotFoundException;
+}
[17/64] [abbrv] git commit: Merge branch '1.4.6-SNAPSHOT' into
1.5.2-SNAPSHOT
Posted by ct...@apache.org.
Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Conflicts:
core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java
core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java
core/src/main/java/org/apache/accumulo/core/iterators/FirstEntryInRowIterator.java
core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
server/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
server/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
server/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
src/core/src/main/java/org/apache/accumulo/core/file/map/MapFileUtil.java
src/server/src/main/java/org/apache/accumulo/server/util/DumpMapFile.java
src/server/src/main/java/org/apache/accumulo/server/util/DumpTabletsOnServer.java
test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/92613388
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/92613388
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/92613388
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 92613388919b6fc138e829382cc1cbb6647faa31
Parents: 5363d78 c8e165a
Author: Christopher Tubbs <ct...@apache.org>
Authored: Wed Apr 9 13:14:57 2014 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Wed Apr 9 13:14:57 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/Constants.java | 1 -
.../core/client/ClientSideIteratorScanner.java | 2 -
.../apache/accumulo/core/client/Connector.java | 2 -
.../accumulo/core/client/IteratorSetting.java | 4 -
.../accumulo/core/client/RowIterator.java | 4 -
.../accumulo/core/client/ScannerBase.java | 2 +-
.../accumulo/core/client/ZooKeeperInstance.java | 2 -
.../core/client/admin/ActiveCompaction.java | 1 -
.../core/client/admin/InstanceOperations.java | 12 -
.../client/admin/InstanceOperationsImpl.java | 33 --
.../core/client/admin/TableOperations.java | 32 --
.../core/client/admin/TableOperationsImpl.java | 4 -
.../core/client/impl/OfflineScanner.java | 6 -
.../core/client/impl/ThriftTransportPool.java | 16 +-
.../client/mapred/AccumuloOutputFormat.java | 1 -
.../core/client/mapred/InputFormatBase.java | 1 -
.../client/mapreduce/AccumuloOutputFormat.java | 1 -
.../core/client/mapreduce/InputFormatBase.java | 1 -
.../mapreduce/lib/util/ConfiguratorBase.java | 1 -
.../core/client/mock/MockBatchDeleter.java | 4 -
.../client/mock/MockInstanceOperations.java | 43 ---
.../apache/accumulo/core/data/ColumnUpdate.java | 1 -
.../java/org/apache/accumulo/core/data/Key.java | 7 +-
.../apache/accumulo/core/data/KeyExtent.java | 6 +-
.../org/apache/accumulo/core/data/Range.java | 17 +-
.../accumulo/core/file/rfile/BlockIndex.java | 5 -
.../accumulo/core/file/rfile/bcfile/BCFile.java | 14 +-
.../core/file/rfile/bcfile/ByteArray.java | 2 -
.../accumulo/core/file/rfile/bcfile/Chunk.java | 2 -
.../accumulo/core/file/rfile/bcfile/TFile.java | 44 ---
.../core/file/rfile/bcfile/TFileDumper.java | 1 -
.../accumulo/core/file/rfile/bcfile/Utils.java | 11 -
.../core/iterators/TypedValueCombiner.java | 6 -
.../core/iterators/ValueFormatException.java | 6 -
.../core/iterators/system/MapFileIterator.java | 8 +-
.../core/iterators/user/GrepIterator.java | 3 -
.../iterators/user/IntersectingIterator.java | 10 -
.../accumulo/core/iterators/user/RowFilter.java | 1 -
.../iterators/user/TransformingIterator.java | 166 +++++-----
.../core/iterators/user/VersioningIterator.java | 3 -
.../accumulo/core/security/SecurityUtil.java | 1 -
.../core/security/crypto/CryptoModule.java | 2 -
.../security/crypto/CryptoModuleFactory.java | 1 -
.../core/client/impl/ScannerOptionsTest.java | 2 -
.../client/mapred/AccumuloInputFormatTest.java | 4 -
.../mapreduce/AccumuloInputFormatTest.java | 6 -
.../shell/command/FormatterCommandTest.java | 15 +-
.../simple/client/RandomBatchScanner.java | 5 -
.../simple/client/RandomBatchWriter.java | 4 -
.../simple/client/SequentialBatchWriter.java | 5 -
.../simple/client/TraceDumpExample.java | 9 +-
.../examples/simple/dirlist/QueryUtil.java | 3 -
.../examples/simple/mapreduce/NGramIngest.java | 3 -
.../examples/simple/mapreduce/TableToFile.java | 1 -
.../accumulo/examples/simple/shard/Query.java | 3 -
.../minicluster/MiniAccumuloCluster.java | 10 -
.../accumulo/proxy/TestProxyReadWrite.java | 10 -
.../accumulo/server/conf/ConfigSanityCheck.java | 3 -
.../accumulo/server/logger/LogReader.java | 1 -
.../master/balancer/ChaoticLoadBalancer.java | 8 -
.../server/master/balancer/TabletBalancer.java | 2 -
.../server/master/state/TabletStateStore.java | 8 +-
.../server/master/tableOps/TraceRepo.java | 25 --
.../server/metanalysis/LogFileOutputFormat.java | 4 -
.../server/metanalysis/PrintEvents.java | 5 +-
.../server/metrics/AbstractMetricsImpl.java | 4 -
.../security/handler/InsecurePermHandler.java | 42 ---
.../server/security/handler/ZKAuthorizor.java | 7 +-
.../server/security/handler/ZKPermHandler.java | 6 +-
.../accumulo/server/tabletserver/MemValue.java | 4 +-
.../server/tabletserver/TabletServer.java | 8 +-
.../server/util/AddFilesWithMissingEntries.java | 1 -
.../accumulo/server/util/DumpZookeeper.java | 3 -
.../server/util/FindOfflineTablets.java | 3 -
.../accumulo/server/util/LoginProperties.java | 3 -
.../accumulo/server/util/MetadataTable.java | 305 +++++++++----------
.../accumulo/server/util/RestoreZookeeper.java | 4 -
.../accumulo/server/util/SendLogToChainsaw.java | 1 -
.../accumulo/server/util/TServerUtils.java | 7 +-
.../accumulo/server/util/TableDiskUsage.java | 3 -
.../accumulo/server/util/TabletServerLocks.java | 3 -
.../start/classloader/AccumuloClassLoader.java | 3 -
.../start/classloader/vfs/ContextManager.java | 2 -
.../vfs/PostDelegatingVFSClassLoader.java | 7 +-
.../vfs/providers/HdfsFileSystem.java | 5 -
.../apache/accumulo/test/GetMasterStats.java | 9 -
.../accumulo/test/NativeMapPerformanceTest.java | 3 -
.../accumulo/test/NativeMapStressTest.java | 3 -
.../test/continuous/ContinuousMoru.java | 1 -
.../test/continuous/ContinuousVerify.java | 1 -
.../test/functional/CacheTestClean.java | 3 -
.../accumulo/test/functional/RunTests.java | 4 -
.../metadata/MetadataBatchScanTest.java | 3 -
.../test/performance/thrift/NullTserver.java | 8 +-
.../accumulo/test/randomwalk/Framework.java | 3 -
.../apache/accumulo/test/randomwalk/Node.java | 1 -
.../randomwalk/concurrent/CheckBalance.java | 5 +-
.../instrument/receivers/ZooSpanClient.java | 10 -
98 files changed, 284 insertions(+), 817 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/Constants.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/Constants.java
index 095319e,0000000..66c4034
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/Constants.java
+++ b/core/src/main/java/org/apache/accumulo/core/Constants.java
@@@ -1,213 -1,0 +1,212 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core;
+
+import java.nio.charset.Charset;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+
+public class Constants {
+ public static final Charset UTF8 = Charset.forName("UTF-8");
+ public static final String VERSION = FilteredConstants.VERSION;
+
+ // versions should never be negative
+ public static final Integer WIRE_VERSION = 2;
+ public static final int DATA_VERSION = 5;
+ public static final int PREV_DATA_VERSION = 4;
+
+ // Zookeeper locations
+ public static final String ZROOT = "/accumulo";
+ public static final String ZINSTANCES = "/instances";
+
+ public static final String ZTABLES = "/tables";
+ public static final byte[] ZTABLES_INITIAL_ID = new byte[] {'0'};
+ public static final String ZTABLE_NAME = "/name";
+ public static final String ZTABLE_CONF = "/conf";
+ public static final String ZTABLE_STATE = "/state";
+ public static final String ZTABLE_FLUSH_ID = "/flush-id";
+ public static final String ZTABLE_COMPACT_ID = "/compact-id";
+ public static final String ZTABLE_COMPACT_CANCEL_ID = "/compact-cancel-id";
+
+ public static final String ZROOT_TABLET = "/root_tablet";
+ public static final String ZROOT_TABLET_LOCATION = ZROOT_TABLET + "/location";
+ public static final String ZROOT_TABLET_FUTURE_LOCATION = ZROOT_TABLET + "/future_location";
+ public static final String ZROOT_TABLET_LAST_LOCATION = ZROOT_TABLET + "/lastlocation";
+ public static final String ZROOT_TABLET_WALOGS = ZROOT_TABLET + "/walogs";
+
+ public static final String ZMASTERS = "/masters";
+ public static final String ZMASTER_LOCK = ZMASTERS + "/lock";
+ public static final String ZMASTER_GOAL_STATE = ZMASTERS + "/goal_state";
+ public static final String ZGC = "/gc";
+ public static final String ZGC_LOCK = ZGC + "/lock";
+
+ public static final String ZMONITOR = "/monitor";
+ public static final String ZMONITOR_LOCK = ZMONITOR + "/lock";
+ public static final String ZMONITOR_HTTP_ADDR = ZMONITOR + "/http_addr";
+ public static final String ZMONITOR_LOG4J_ADDR = ZMONITOR + "/log4j_addr";
+
+ public static final String ZCONFIG = "/config";
+
+ public static final String ZTSERVERS = "/tservers";
+
+ public static final String ZDEAD = "/dead";
+ public static final String ZDEADTSERVERS = "/dead/tservers";
+
+ public static final String ZTRACERS = "/tracers";
+
+ public static final String ZPROBLEMS = "/problems";
+ public static final String ZUSERS = "/users";
+
+ public static final String BULK_ARBITRATOR_TYPE = "bulkTx";
+
+ public static final String ZFATE = "/fate";
+
+ public static final String ZNEXT_FILE = "/next_file";
+
+ public static final String ZBULK_FAILED_COPYQ = "/bulk_failed_copyq";
+
+ public static final String ZHDFS_RESERVATIONS = "/hdfs_reservations";
+ public static final String ZRECOVERY = "/recovery";
+
+ public static final String METADATA_TABLE_ID = "!0";
+ public static final String METADATA_TABLE_NAME = "!METADATA";
+ public static final String DEFAULT_TABLET_LOCATION = "/default_tablet";
+ public static final String TABLE_TABLET_LOCATION = "/table_info";
+ public static final String ZTABLE_LOCKS = "/table_locks";
+
+ // reserved keyspace is any row that begins with a tilde '~' character
+ public static final Key METADATA_RESERVED_KEYSPACE_START_KEY = new Key(new Text(new byte[] {'~'}));
+ public static final Key METADATA_RESERVED_KEYSPACE_STOP_KEY = new Key(new Text(new byte[] {'~' + 1}));
+ public static final Range METADATA_RESERVED_KEYSPACE = new Range(METADATA_RESERVED_KEYSPACE_START_KEY, true, METADATA_RESERVED_KEYSPACE_STOP_KEY, false);
+ public static final String METADATA_DELETE_FLAG_PREFIX = "~del";
+ public static final String METADATA_DELETE_FLAG_FOR_METADATA_PREFIX = "!!" + METADATA_DELETE_FLAG_PREFIX;
+ public static final Range METADATA_DELETES_KEYSPACE = new Range(new Key(new Text(METADATA_DELETE_FLAG_PREFIX)), true, new Key(new Text("~dem")), false);
+ public static final Range METADATA_DELETES_FOR_METADATA_KEYSPACE = new Range(new Key(new Text(METADATA_DELETE_FLAG_FOR_METADATA_PREFIX)), true, new Key(new Text("!!~dem")), false);
+ public static final String METADATA_BLIP_FLAG_PREFIX = "~blip"; // BLIP = bulk load in progress
+ public static final Range METADATA_BLIP_KEYSPACE = new Range(new Key(new Text(METADATA_BLIP_FLAG_PREFIX)), true, new Key(new Text("~bliq")), false);
+
+ public static final Text METADATA_SERVER_COLUMN_FAMILY = new Text("srv");
+ public static final Text METADATA_TABLET_COLUMN_FAMILY = new Text("~tab"); // this needs to sort after all other column families for that tablet
+ public static final Text METADATA_CURRENT_LOCATION_COLUMN_FAMILY = new Text("loc");
+ public static final Text METADATA_FUTURE_LOCATION_COLUMN_FAMILY = new Text("future");
+ public static final Text METADATA_LAST_LOCATION_COLUMN_FAMILY = new Text("last");
+ public static final Text METADATA_BULKFILE_COLUMN_FAMILY = new Text("loaded"); // temporary marker that indicates a tablet loaded a bulk file
+ public static final Text METADATA_CLONED_COLUMN_FAMILY = new Text("!cloned"); // temporary marker that indicates a tablet was successfully cloned
+
+ // README : very important that prevRow sort last to avoid race conditions between
+ // garbage collector and split
+ public static final ColumnFQ METADATA_PREV_ROW_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("~pr")); // this needs to sort after everything
+ // else for that tablet
+ public static final ColumnFQ METADATA_OLD_PREV_ROW_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("oldprevrow"));
+ public static final ColumnFQ METADATA_DIRECTORY_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("dir"));
+ public static final ColumnFQ METADATA_TIME_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("time"));
+ public static final ColumnFQ METADATA_FLUSH_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("flush"));
+ public static final ColumnFQ METADATA_COMPACT_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("compact"));
+ public static final ColumnFQ METADATA_SPLIT_RATIO_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("splitRatio"));
+ public static final ColumnFQ METADATA_LOCK_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("lock"));
+
+ public static final Text METADATA_DATAFILE_COLUMN_FAMILY = new Text("file");
+ public static final Text METADATA_SCANFILE_COLUMN_FAMILY = new Text("scan");
+ public static final Text METADATA_LOG_COLUMN_FAMILY = new Text("log");
+ public static final Text METADATA_CHOPPED_COLUMN_FAMILY = new Text("chopped");
+ public static final ColumnFQ METADATA_CHOPPED_COLUMN = new ColumnFQ(METADATA_CHOPPED_COLUMN_FAMILY, new Text("chopped"));
+
+ public static final Range NON_ROOT_METADATA_KEYSPACE = new Range(
+ new Key(KeyExtent.getMetadataEntry(new Text(METADATA_TABLE_ID), null)).followingKey(PartialKey.ROW), true, METADATA_RESERVED_KEYSPACE_START_KEY, false);
+ public static final Range METADATA_KEYSPACE = new Range(new Key(new Text(METADATA_TABLE_ID)), true, METADATA_RESERVED_KEYSPACE_START_KEY, false);
+
+ public static final KeyExtent ROOT_TABLET_EXTENT = new KeyExtent(new Text(METADATA_TABLE_ID), KeyExtent.getMetadataEntry(new Text(METADATA_TABLE_ID), null),
+ null);
+ public static final Range METADATA_ROOT_TABLET_KEYSPACE = new Range(ROOT_TABLET_EXTENT.getMetadataEntry(), false, KeyExtent.getMetadataEntry(new Text(
+ METADATA_TABLE_ID), null), true);
+
+ public static final String VALUE_ENCODING = "UTF-8";
+
+ public static final String BULK_PREFIX = "b-";
+ public static final String OLD_BULK_PREFIX = "bulk_";
+
+ // note: all times are in milliseconds
+
+ public static final int SCAN_BATCH_SIZE = 1000; // this affects the table client caching of metadata
+
+ public static final long MIN_MASTER_LOOP_TIME = 1000;
+ public static final int MASTER_TABLETSERVER_CONNECTION_TIMEOUT = 3000;
+ public static final long CLIENT_SLEEP_BEFORE_RECONNECT = 1000;
+
+ // Security configuration
+ public static final String PW_HASH_ALGORITHM = "SHA-256";
+
+ // Representation of an empty set of authorizations
+ // (used throughout the code, because scans of metadata table and many tests do not set record-level visibility)
+ public static final Authorizations NO_AUTHS = new Authorizations();
+
+ public static final int DEFAULT_MINOR_COMPACTION_MAX_SLEEP_TIME = 60 * 3; // in seconds
+
+ public static final int MAX_DATA_TO_PRINT = 64;
+ public static final int CLIENT_RETRIES = 5;
+ public static final int TSERV_MINC_MAXCONCURRENT_NUMWAITING_MULTIPLIER = 2;
+ public static final String CORE_PACKAGE_NAME = "org.apache.accumulo.core";
+ public static final String OLD_PACKAGE_NAME = "cloudbase";
+ public static final String VALID_TABLE_NAME_REGEX = "^\\w+$";
+ public static final String MAPFILE_EXTENSION = "map";
+ public static final String GENERATED_TABLET_DIRECTORY_PREFIX = "t-";
+
+ public static final String EXPORT_METADATA_FILE = "metadata.bin";
+ public static final String EXPORT_TABLE_CONFIG_FILE = "table_config.txt";
+ public static final String EXPORT_FILE = "exportMetadata.zip";
+ public static final String EXPORT_INFO_FILE = "accumulo_export_info.txt";
+
+ public static String getBaseDir(final AccumuloConfiguration conf) {
+ return conf.get(Property.INSTANCE_DFS_DIR);
+ }
+
+ public static String getTablesDir(final AccumuloConfiguration conf) {
+ return getBaseDir(conf) + "/tables";
+ }
+
+ public static String getRecoveryDir(final AccumuloConfiguration conf) {
+ return getBaseDir(conf) + "/recovery";
+ }
+
+ public static Path getDataVersionLocation(final AccumuloConfiguration conf) {
+ return new Path(getBaseDir(conf) + "/version");
+ }
+
+ public static String getMetadataTableDir(final AccumuloConfiguration conf) {
+ return getTablesDir(conf) + "/" + METADATA_TABLE_ID;
+ }
+
+ public static String getRootTabletDir(final AccumuloConfiguration conf) {
+ return getMetadataTableDir(conf) + ZROOT_TABLET;
+ }
+
+ /**
- * @param conf
+ * @return The write-ahead log directory.
+ */
+ public static String getWalDirectory(final AccumuloConfiguration conf) {
+ return getBaseDir(conf) + "/wal";
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
index 3085f56,0000000..fbf8670
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
@@@ -1,257 -1,0 +1,255 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.impl.ScannerOptions;
+import org.apache.accumulo.core.client.mock.IteratorAdapter;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Column;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.hadoop.io.Text;
+
+/**
+ * A scanner that instantiates iterators on the client side instead of on the tablet server. This can be useful for testing iterators or in cases where you
+ * don't want iterators affecting the performance of tablet servers.<br>
+ * <br>
+ * Suggested usage:<br>
+ * <code>Scanner scanner = new ClientSideIteratorScanner(connector.createScanner(tableName, authorizations));</code><br>
+ * <br>
+ * Iterators added to this scanner will be run in the client JVM. Separate scan iterators can be run on the server side and client side by adding iterators to
+ * the source scanner (which will execute server side) and to the client side scanner (which will execute client side).
+ */
+public class ClientSideIteratorScanner extends ScannerOptions implements Scanner {
+ private int size;
+
+ private Range range;
+ private boolean isolated = false;
+
+ /**
+ * A class that wraps a Scanner in a SortedKeyValueIterator so that other accumulo iterators can use it as a source.
+ */
+ public class ScannerTranslator implements SortedKeyValueIterator<Key,Value> {
+ protected Scanner scanner;
+ Iterator<Entry<Key,Value>> iter;
+ Entry<Key,Value> top = null;
+
+ /**
+ * Constructs an accumulo iterator from a scanner.
+ *
+ * @param scanner
+ * the scanner to iterate over
+ */
+ public ScannerTranslator(final Scanner scanner) {
+ this.scanner = scanner;
+ }
+
+ @Override
+ public void init(final SortedKeyValueIterator<Key,Value> source, final Map<String,String> options, final IteratorEnvironment env) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean hasTop() {
+ return top != null;
+ }
+
+ @Override
+ public void next() throws IOException {
+ if (iter.hasNext())
+ top = iter.next();
+ else
+ top = null;
+ }
+
+ @Override
+ public void seek(final Range range, final Collection<ByteSequence> columnFamilies, final boolean inclusive) throws IOException {
+ if (!inclusive && columnFamilies.size() > 0) {
+ throw new IllegalArgumentException();
+ }
+ scanner.setRange(range);
+ scanner.clearColumns();
+ for (ByteSequence colf : columnFamilies) {
+ scanner.fetchColumnFamily(new Text(colf.toArray()));
+ }
+ iter = scanner.iterator();
+ next();
+ }
+
+ @Override
+ public Key getTopKey() {
+ return top.getKey();
+ }
+
+ @Override
+ public Value getTopValue() {
+ return top.getValue();
+ }
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(final IteratorEnvironment env) {
+ return new ScannerTranslator(scanner);
+ }
+ }
+
+ private ScannerTranslator smi;
+
+ /**
+ * Constructs a scanner that can execute client-side iterators.
+ *
+ * @param scanner
+ * the source scanner
+ */
+ public ClientSideIteratorScanner(final Scanner scanner) {
+ smi = new ScannerTranslator(scanner);
+ this.range = scanner.getRange();
+ this.size = scanner.getBatchSize();
+ this.timeOut = scanner.getTimeout(TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Sets the source Scanner.
- *
- * @param scanner
+ */
+ public void setSource(final Scanner scanner) {
+ smi = new ScannerTranslator(scanner);
+ }
+
+ @Override
+ public Iterator<Entry<Key,Value>> iterator() {
+ smi.scanner.setBatchSize(size);
+ smi.scanner.setTimeout(timeOut, TimeUnit.MILLISECONDS);
+ if (isolated)
+ smi.scanner.enableIsolation();
+ else
+ smi.scanner.disableIsolation();
+
+ final TreeMap<Integer,IterInfo> tm = new TreeMap<Integer,IterInfo>();
+
+ for (IterInfo iterInfo : serverSideIteratorList) {
+ tm.put(iterInfo.getPriority(), iterInfo);
+ }
+
+ SortedKeyValueIterator<Key,Value> skvi;
+ try {
+ skvi = IteratorUtil.loadIterators(smi, tm.values(), serverSideIteratorOptions, new IteratorEnvironment() {
+ @Override
+ public SortedKeyValueIterator<Key,Value> reserveMapFileReader(final String mapFileName) throws IOException {
+ return null;
+ }
+
+ @Override
+ public AccumuloConfiguration getConfig() {
+ return null;
+ }
+
+ @Override
+ public IteratorScope getIteratorScope() {
+ return null;
+ }
+
+ @Override
+ public boolean isFullMajorCompaction() {
+ return false;
+ }
+
+ @Override
+ public void registerSideChannel(final SortedKeyValueIterator<Key,Value> iter) {}
+ }, false, null);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ final Set<ByteSequence> colfs = new TreeSet<ByteSequence>();
+ for (Column c : this.getFetchedColumns()) {
+ colfs.add(new ArrayByteSequence(c.getColumnFamily()));
+ }
+
+ try {
+ skvi.seek(range, colfs, true);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ return new IteratorAdapter(skvi);
+ }
+
+ @Deprecated
+ @Override
+ public void setTimeOut(int timeOut) {
+ if (timeOut == Integer.MAX_VALUE)
+ setTimeout(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
+ else
+ setTimeout(timeOut, TimeUnit.SECONDS);
+ }
+
+ @Deprecated
+ @Override
+ public int getTimeOut() {
+ long timeout = getTimeout(TimeUnit.SECONDS);
+ if (timeout >= Integer.MAX_VALUE)
+ return Integer.MAX_VALUE;
+ return (int) timeout;
+ }
+
+ @Override
+ public void setRange(final Range range) {
+ this.range = range;
+ }
+
+ @Override
+ public Range getRange() {
+ return range;
+ }
+
+ @Override
+ public void setBatchSize(final int size) {
+ this.size = size;
+ }
+
+ @Override
+ public int getBatchSize() {
+ return size;
+ }
+
+ @Override
+ public void enableIsolation() {
+ this.isolated = true;
+ }
+
+ @Override
+ public void disableIsolation() {
+ this.isolated = false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/Connector.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/Connector.java
index d2e7321,0000000..3189d44
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/Connector.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/Connector.java
@@@ -1,210 -1,0 +1,208 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
+import org.apache.accumulo.core.client.admin.InstanceOperations;
+import org.apache.accumulo.core.client.admin.SecurityOperations;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.security.Authorizations;
+
+/**
+ * Connector connects to an Accumulo instance and allows the user to request readers and writers for the instance as well as various objects that permit
+ * administrative operations.
+ *
+ * The Connector enforces security on the client side by forcing all API calls to be accompanied by user credentials.
+ */
+public abstract class Connector {
+
+ /**
+ * Factory method to create a BatchScanner connected to Accumulo.
+ *
+ * @param tableName
+ * the name of the table to query
+ * @param authorizations
+ * A set of authorization labels that will be checked against the column visibility of each key in order to filter data. The authorizations passed in
+ * must be a subset of the accumulo user's set of authorizations. If the accumulo user has authorizations (A1, A2) and authorizations (A2, A3) are
+ * passed, then an exception will be thrown.
+ * @param numQueryThreads
+ * the number of concurrent threads to spawn for querying
+ *
+ * @return BatchScanner object for configuring and querying
+ * @throws TableNotFoundException
+ * when the specified table doesn't exist
+ */
+ public abstract BatchScanner createBatchScanner(String tableName, Authorizations authorizations, int numQueryThreads) throws TableNotFoundException;
+
+ /**
+ * Factory method to create a BatchDeleter connected to Accumulo.
+ *
+ * @param tableName
+ * the name of the table to query and delete from
+ * @param authorizations
+ * A set of authorization labels that will be checked against the column visibility of each key in order to filter data. The authorizations passed in
+ * must be a subset of the accumulo user's set of authorizations. If the accumulo user has authorizations (A1, A2) and authorizations (A2, A3) are
+ * passed, then an exception will be thrown.
+ * @param numQueryThreads
+ * the number of concurrent threads to spawn for querying
+ * @param maxMemory
+ * size in bytes of the maximum memory to batch before writing
+ * @param maxLatency
+ * size in milliseconds; set to 0 or Long.MAX_VALUE to allow the maximum time to hold a batch before writing
+ * @param maxWriteThreads
+ * the maximum number of threads to use for writing data to the tablet servers
+ *
+ * @return BatchDeleter object for configuring and deleting
+ * @throws TableNotFoundException
+ * when the specified table doesn't exist
+ * @deprecated since 1.5.0; Use {@link #createBatchDeleter(String, Authorizations, int, BatchWriterConfig)} instead.
+ */
+ @Deprecated
+ public abstract BatchDeleter createBatchDeleter(String tableName, Authorizations authorizations, int numQueryThreads, long maxMemory, long maxLatency,
+ int maxWriteThreads) throws TableNotFoundException;
+
+ /**
+ *
+ * @param tableName
+ * the name of the table to query and delete from
+ * @param authorizations
+ * A set of authorization labels that will be checked against the column visibility of each key in order to filter data. The authorizations passed in
+ * must be a subset of the accumulo user's set of authorizations. If the accumulo user has authorizations (A1, A2) and authorizations (A2, A3) are
+ * passed, then an exception will be thrown.
+ * @param numQueryThreads
+ * the number of concurrent threads to spawn for querying
+ * @param config
+ * configuration used to create batch writer
+ * @return BatchDeleter object for configuring and deleting
- * @throws TableNotFoundException
+ * @since 1.5.0
+ */
+
+ public abstract BatchDeleter createBatchDeleter(String tableName, Authorizations authorizations, int numQueryThreads, BatchWriterConfig config)
+ throws TableNotFoundException;
+
+ /**
+ * Factory method to create a BatchWriter connected to Accumulo.
+ *
+ * @param tableName
+ * the name of the table to insert data into
+ * @param maxMemory
+ * size in bytes of the maximum memory to batch before writing
+ * @param maxLatency
+ * time in milliseconds; set to 0 or Long.MAX_VALUE to allow the maximum time to hold a batch before writing
+ * @param maxWriteThreads
+ * the maximum number of threads to use for writing data to the tablet servers
+ *
+ * @return BatchWriter object for configuring and writing data to
+ * @throws TableNotFoundException
+ * when the specified table doesn't exist
+ * @deprecated since 1.5.0; Use {@link #createBatchWriter(String, BatchWriterConfig)} instead.
+ */
+ @Deprecated
+ public abstract BatchWriter createBatchWriter(String tableName, long maxMemory, long maxLatency, int maxWriteThreads) throws TableNotFoundException;
+
+ /**
+ * Factory method to create a BatchWriter connected to Accumulo.
+ *
+ * @param tableName
+ * the name of the table to insert data into
+ * @param config
+ * configuration used to create batch writer
+ * @return BatchWriter object for configuring and writing data to
- * @throws TableNotFoundException
+ * @since 1.5.0
+ */
+
+ public abstract BatchWriter createBatchWriter(String tableName, BatchWriterConfig config) throws TableNotFoundException;
+
+ /**
+ * Factory method to create a Multi-Table BatchWriter connected to Accumulo. Multi-table batch writers can queue data for multiple tables, which is good for
+ * ingesting data into multiple tables from the same source
+ *
+ * @param maxMemory
+ * size in bytes of the maximum memory to batch before writing
+ * @param maxLatency
+ * size in milliseconds; set to 0 or Long.MAX_VALUE to allow the maximum time to hold a batch before writing
+ * @param maxWriteThreads
+ * the maximum number of threads to use for writing data to the tablet servers
+ *
+ * @return MultiTableBatchWriter object for configuring and writing data to
+ * @deprecated since 1.5.0; Use {@link #createMultiTableBatchWriter(BatchWriterConfig)} instead.
+ */
+ @Deprecated
+ public abstract MultiTableBatchWriter createMultiTableBatchWriter(long maxMemory, long maxLatency, int maxWriteThreads);
+
+ /**
+ * Factory method to create a Multi-Table BatchWriter connected to Accumulo. Multi-table batch writers can queue data for multiple tables. Also data for
+ * multiple tables can be sent to a server in a single batch. Its an efficient way to ingest data into multiple tables from a single process.
+ *
+ * @param config
+ * configuration used to create multi-table batch writer
+ * @return MultiTableBatchWriter object for configuring and writing data to
+ * @since 1.5.0
+ */
+
+ public abstract MultiTableBatchWriter createMultiTableBatchWriter(BatchWriterConfig config);
+
+ /**
+ * Factory method to create a Scanner connected to Accumulo.
+ *
+ * @param tableName
+ * the name of the table to query data from
+ * @param authorizations
+ * A set of authorization labels that will be checked against the column visibility of each key in order to filter data. The authorizations passed in
+ * must be a subset of the accumulo user's set of authorizations. If the accumulo user has authorizations (A1, A2) and authorizations (A2, A3) are
+ * passed, then an exception will be thrown.
+ *
+ * @return Scanner object for configuring and querying data with
+ * @throws TableNotFoundException
+ * when the specified table doesn't exist
+ */
+ public abstract Scanner createScanner(String tableName, Authorizations authorizations) throws TableNotFoundException;
+
+ /**
+ * Accessor method for internal instance object.
+ *
+ * @return the internal instance object
+ */
+ public abstract Instance getInstance();
+
+ /**
+ * Get the current user for this connector
+ *
+ * @return the user name
+ */
+ public abstract String whoami();
+
+ /**
+ * Retrieves a TableOperations object to perform table functions, such as create and delete.
+ *
+ * @return an object to manipulate tables
+ */
+ public abstract TableOperations tableOperations();
+
+ /**
+ * Retrieves a SecurityOperations object to perform user security operations, such as creating users.
+ *
+ * @return an object to modify users and permissions
+ */
+ public abstract SecurityOperations securityOperations();
+
+ /**
+ * Retrieves an InstanceOperations object to modify instance configuration.
+ *
+ * @return an object to modify instance configuration
+ */
+ public abstract InstanceOperations instanceOperations();
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
index 85e996a,0000000..e58a1be
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
@@@ -1,387 -1,0 +1,383 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+
+/**
+ * Configure an iterator for minc, majc, and/or scan. By default, IteratorSetting will be configured for scan.
+ *
+ * Every iterator has a priority, a name, a class, a set of scopes, and configuration parameters.
+ *
+ * A typical use case configured for scan:
+ *
+ * <pre>
+ * IteratorSetting cfg = new IteratorSetting(priority, "myIter", MyIterator.class);
+ * MyIterator.addOption(cfg, 42);
+ * scanner.addScanIterator(cfg);
+ * </pre>
+ */
+public class IteratorSetting implements Writable {
+ private int priority;
+ private String name;
+ private String iteratorClass;
+ private Map<String,String> properties;
+
+ /**
+ * Get layer at which this iterator applies. See {@link #setPriority(int)} for how the priority is used.
+ *
+ * @return the priority of this Iterator
+ */
+ public int getPriority() {
+ return priority;
+ }
+
+ /**
+ * Set layer at which this iterator applies.
+ *
+ * @param priority
+ * determines the order in which iterators are applied (system iterators are always applied first, then user-configured iterators, lowest priority
+ * first)
+ */
+ public void setPriority(int priority) {
+ ArgumentChecker.strictlyPositive(priority);
+ this.priority = priority;
+ }
+
+ /**
+ * Get the iterator's name.
+ *
+ * @return the name of the iterator
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Set the iterator's name. Must be a simple alphanumeric identifier.
- *
- * @param name
+ */
+ public void setName(String name) {
+ ArgumentChecker.notNull(name);
+ this.name = name;
+ }
+
+ /**
+ * Get the name of the class that implements the iterator.
+ *
+ * @return the iterator's class name
+ */
+ public String getIteratorClass() {
+ return iteratorClass;
+ }
+
+ /**
+ * Set the name of the class that implements the iterator. The class does not have to be present on the client, but it must be available to all tablet
+ * servers.
- *
- * @param iteratorClass
+ */
+ public void setIteratorClass(String iteratorClass) {
+ ArgumentChecker.notNull(iteratorClass);
+ this.iteratorClass = iteratorClass;
+ }
+
+ /**
+ * Constructs an iterator setting configured for the scan scope with no parameters. (Parameters can be added later.)
+ *
+ * @param priority
+ * the priority for the iterator (see {@link #setPriority(int)})
+ * @param name
+ * the distinguishing name for the iterator
+ * @param iteratorClass
+ * the fully qualified class name for the iterator
+ */
+ public IteratorSetting(int priority, String name, String iteratorClass) {
+ this(priority, name, iteratorClass, new HashMap<String,String>());
+ }
+
+ /**
+ * Constructs an iterator setting configured for the specified scopes with the specified parameters.
+ *
+ * @param priority
+ * the priority for the iterator (see {@link #setPriority(int)})
+ * @param name
+ * the distinguishing name for the iterator
+ * @param iteratorClass
+ * the fully qualified class name for the iterator
+ * @param properties
+ * any properties for the iterator
+ */
+ public IteratorSetting(int priority, String name, String iteratorClass, Map<String,String> properties) {
+ setPriority(priority);
+ setName(name);
+ setIteratorClass(iteratorClass);
+ this.properties = new HashMap<String,String>();
+ addOptions(properties);
+ }
+
+ /**
+ * Constructs an iterator setting using the given class's SimpleName for the iterator name. The iterator setting will be configured for the scan scope with no
+ * parameters.
+ *
+ * @param priority
+ * the priority for the iterator (see {@link #setPriority(int)})
+ * @param iteratorClass
+ * the class for the iterator
+ */
+ public IteratorSetting(int priority, Class<? extends SortedKeyValueIterator<Key,Value>> iteratorClass) {
+ this(priority, iteratorClass.getSimpleName(), iteratorClass.getName());
+ }
+
+ /**
+ *
+ * Constructs an iterator setting using the given class's SimpleName for the iterator name and configured for the specified scopes with the specified
+ * parameters.
+ *
+ * @param priority
+ * the priority for the iterator (see {@link #setPriority(int)})
+ * @param iteratorClass
+ * the class for the iterator
+ * @param properties
+ * any properties for the iterator
+ */
+ public IteratorSetting(int priority, Class<? extends SortedKeyValueIterator<Key,Value>> iteratorClass, Map<String,String> properties) {
+ this(priority, iteratorClass.getSimpleName(), iteratorClass.getName(), properties);
+ }
+
+ /**
+ * Constructs an iterator setting configured for the scan scope with no parameters.
+ *
+ * @param priority
+ * the priority for the iterator (see {@link #setPriority(int)})
+ * @param name
+ * the distinguishing name for the iterator
+ * @param iteratorClass
+ * the class for the iterator
+ */
+ public IteratorSetting(int priority, String name, Class<? extends SortedKeyValueIterator<Key,Value>> iteratorClass) {
+ this(priority, name, iteratorClass.getName());
+ }
+
+ /**
+ * @since 1.5.0
+ */
+ public IteratorSetting(DataInput din) throws IOException {
+ this.properties = new HashMap<String,String>();
+ this.readFields(din);
+ }
+
+ /**
+ * Add another option to the iterator.
+ *
+ * @param option
+ * the name of the option
+ * @param value
+ * the value of the option
+ */
+ public void addOption(String option, String value) {
+ ArgumentChecker.notNull(option, value);
+ properties.put(option, value);
+ }
+
+ /**
+ * Remove an option from the iterator.
+ *
+ * @param option
+ * the name of the option
+ * @return the value previously associated with the option, or null if no such option existed
+ */
+ public String removeOption(String option) {
+ ArgumentChecker.notNull(option);
+ return properties.remove(option);
+ }
+
+ /**
+ * Add many options to the iterator.
+ *
+ * @param propertyEntries
+ * a set of entries to add to the options
+ */
+ public void addOptions(Set<Entry<String,String>> propertyEntries) {
+ ArgumentChecker.notNull(propertyEntries);
+ for (Entry<String,String> keyValue : propertyEntries) {
+ addOption(keyValue.getKey(), keyValue.getValue());
+ }
+ }
+
+ /**
+ * Add many options to the iterator.
+ *
+ * @param properties
+ * a map of entries to add to the options
+ */
+ public void addOptions(Map<String,String> properties) {
+ ArgumentChecker.notNull(properties);
+ addOptions(properties.entrySet());
+ }
+
+ /**
+ * Get the configuration parameters for this iterator.
+ *
+ * @return the properties
+ */
+ public Map<String,String> getOptions() {
+ return Collections.unmodifiableMap(properties);
+ }
+
+ /**
+ * Remove all options from the iterator.
+ */
+ public void clearOptions() {
+ properties.clear();
+ }
+
+ /**
+ * @see java.lang.Object#hashCode()
+ */
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((iteratorClass == null) ? 0 : iteratorClass.hashCode());
+ result = prime * result + ((name == null) ? 0 : name.hashCode());
+ result = prime * result + priority;
+ result = prime * result + ((properties == null) ? 0 : properties.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (!(obj instanceof IteratorSetting))
+ return false;
+ IteratorSetting other = (IteratorSetting) obj;
+ if (iteratorClass == null) {
+ if (other.iteratorClass != null)
+ return false;
+ } else if (!iteratorClass.equals(other.iteratorClass))
+ return false;
+ if (name == null) {
+ if (other.name != null)
+ return false;
+ } else if (!name.equals(other.name))
+ return false;
+ if (priority != other.priority)
+ return false;
+ if (properties == null) {
+ if (other.properties != null)
+ return false;
+ } else if (!properties.equals(other.properties))
+ return false;
+ return true;
+ }
+
+ /**
+ * @see java.lang.Object#toString()
+ */
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("name:");
+ sb.append(name);
+ sb.append(", priority:");
+ sb.append(Integer.toString(priority));
+ sb.append(", class:");
+ sb.append(iteratorClass);
+ sb.append(", properties:");
+ sb.append(properties);
+ return sb.toString();
+ }
+
+ /**
+ * A convenience class for passing column family and column qualifiers to iterator configuration methods.
+ */
+ public static class Column extends Pair<Text,Text> {
+
+ public Column(Text columnFamily, Text columnQualifier) {
+ super(columnFamily, columnQualifier);
+ }
+
+ public Column(Text columnFamily) {
+ super(columnFamily, null);
+ }
+
+ public Column(String columnFamily, String columnQualifier) {
+ super(new Text(columnFamily), new Text(columnQualifier));
+ }
+
+ public Column(String columnFamily) {
+ super(new Text(columnFamily), null);
+ }
+
+ public Text getColumnFamily() {
+ return getFirst();
+ }
+
+ public Text getColumnQualifier() {
+ return getSecond();
+ }
+
+ }
+
+ /**
+ * @since 1.5.0
+ */
+ @Override
+ public void readFields(DataInput din) throws IOException {
+ priority = WritableUtils.readVInt(din);
+ name = WritableUtils.readString(din);
+ iteratorClass = WritableUtils.readString(din);
+ properties.clear();
+ int size = WritableUtils.readVInt(din);
+ while (size > 0) {
+ properties.put(WritableUtils.readString(din), WritableUtils.readString(din));
+ size--;
+ }
+ }
+
+ /**
+ * @since 1.5.0
+ */
+ @Override
+ public void write(DataOutput dout) throws IOException {
+ WritableUtils.writeVInt(dout, priority);
+ WritableUtils.writeString(dout, name);
+ WritableUtils.writeString(dout, iteratorClass);
+ WritableUtils.writeVInt(dout, properties.size());
+ for (Entry<String,String> e : properties.entrySet()) {
+ WritableUtils.writeString(dout, e.getKey());
+ WritableUtils.writeString(dout, e.getValue());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/RowIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/RowIterator.java
index 005f697,0000000..f5e9547
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/RowIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/RowIterator.java
@@@ -1,164 -1,0 +1,160 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.NoSuchElementException;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.PeekingIterator;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Group Key/Value pairs into Iterators over rows. Suggested usage:
+ *
+ * <pre>
+ * RowIterator rowIterator = new RowIterator(connector.createScanner(tableName, authorizations));
+ * </pre>
+ */
+public class RowIterator implements Iterator<Iterator<Entry<Key,Value>>> {
+
+ /**
+ * Iterate over entries in a single row.
+ */
+ private static class SingleRowIter implements Iterator<Entry<Key,Value>> {
+ private PeekingIterator<Entry<Key,Value>> source;
+ private Text currentRow = null;
+ private long count = 0;
+ private boolean disabled = false;
+
+ /**
+ * SingleRowIter must be passed a PeekingIterator so that it can peek at the next entry to see if it belongs in the current row or not.
+ */
+ public SingleRowIter(PeekingIterator<Entry<Key,Value>> source) {
+ this.source = source;
+ if (source.hasNext())
+ currentRow = source.peek().getKey().getRow();
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (disabled)
+ throw new IllegalStateException("SingleRowIter no longer valid");
+ return currentRow != null;
+ }
+
+ @Override
+ public Entry<Key,Value> next() {
+ if (disabled)
+ throw new IllegalStateException("SingleRowIter no longer valid");
+ return _next();
+ }
+
+ private Entry<Key,Value> _next() {
+ if (currentRow == null)
+ throw new NoSuchElementException();
+ count++;
+ Entry<Key,Value> kv = source.next();
+ if (!source.hasNext() || !source.peek().getKey().getRow().equals(currentRow)) {
+ currentRow = null;
+ }
+ return kv;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Get a count of entries read from the row (only equals the number of entries in the row when the row has been read fully).
+ */
+ public long getCount() {
+ return count;
+ }
+
+ /**
+ * Consume the rest of the row. Disables the iterator from future use.
+ */
+ public void consume() {
+ disabled = true;
+ while (currentRow != null)
+ _next();
+ }
+ }
+
+ private final PeekingIterator<Entry<Key,Value>> iter;
+ private long count = 0;
+ private SingleRowIter lastIter = null;
+
+ /**
+ * Create an iterator from an (ordered) sequence of KeyValue pairs.
- *
- * @param iterator
+ */
+ public RowIterator(Iterator<Entry<Key,Value>> iterator) {
+ this.iter = new PeekingIterator<Entry<Key,Value>>(iterator);
+ }
+
+ /**
+ * Create an iterator from an Iterable.
- *
- * @param iterable
+ */
+ public RowIterator(Iterable<Entry<Key,Value>> iterable) {
+ this(iterable.iterator());
+ }
+
+ /**
+ * Returns true if there is at least one more row to get.
+ *
+ * If the last row hasn't been fully read, this method will read through the end of the last row so it can determine if the underlying iterator has a next
+ * row. The last row is disabled from future use.
+ */
+ @Override
+ public boolean hasNext() {
+ if (lastIter != null) {
+ lastIter.consume();
+ count += lastIter.getCount();
+ lastIter = null;
+ }
+ return iter.hasNext();
+ }
+
+ /**
+ * Fetch the next row.
+ */
+ @Override
+ public Iterator<Entry<Key,Value>> next() {
+ if (!hasNext())
+ throw new NoSuchElementException();
+ return lastIter = new SingleRowIter(iter);
+ }
+
+ /**
+ * Unsupported.
+ */
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Get a count of the total number of entries in all rows read so far.
+ */
+ public long getKVCount() {
+ return count;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
index 873a3ad,0000000..7c61d57
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
@@@ -1,130 -1,0 +1,130 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.hadoop.io.Text;
+
+/**
+ * This class hosts configuration methods that are shared between different types of scanners.
+ *
+ */
+public interface ScannerBase extends Iterable<Entry<Key,Value>> {
+
+ /**
+ * Add a server-side scan iterator.
+ *
+ * @param cfg
+ * fully specified scan-time iterator, including all options for the iterator. Any changes to the iterator setting after this call are not propagated
+ * to the stored iterator.
+ * @throws IllegalArgumentException
+ * if the setting conflicts with existing iterators
+ */
+ public void addScanIterator(IteratorSetting cfg);
+
+ /**
+ * Remove an iterator from the list of iterators.
+ *
+ * @param iteratorName
+ * nickname used for the iterator
+ */
+ public void removeScanIterator(String iteratorName);
+
+ /**
+ * Update the options for an iterator. Note that this does <b>not</b> change the iterator options during a scan, it just replaces the given option on a
+ * configured iterator before a scan is started.
+ *
+ * @param iteratorName
+ * the name of the iterator to change
+ * @param key
+ * the name of the option
+ * @param value
+ * the new value for the named option
+ */
+ public void updateScanIteratorOption(String iteratorName, String key, String value);
+
+ /**
+ * Adds a column family to the list of columns that will be fetched by this scanner. By default when no columns have been added the scanner fetches all
+ * columns.
+ *
+ * @param col
+ * the column family to be fetched
+ */
+ public void fetchColumnFamily(Text col);
+
+ /**
+ * Adds a column to the list of columns that will be fetched by this scanner. The column is identified by family and qualifier. By default when no columns
+ * have been added the scanner fetches all columns.
+ *
+ * @param colFam
+ * the column family of the column to be fetched
+ * @param colQual
+ * the column qualifier of the column to be fetched
+ */
+ public void fetchColumn(Text colFam, Text colQual);
+
+ /**
+ * Clears the columns to be fetched (useful for resetting the scanner for reuse). Once cleared, the scanner will fetch all columns.
+ */
+ public void clearColumns();
+
+ /**
+ * Clears scan iterators prior to returning a scanner to the pool.
+ */
+ public void clearScanIterators();
+
+ /**
+ * Returns an iterator over an accumulo table. This iterator uses the options that are currently set for its lifetime, so setting options will have no effect
+ * on existing iterators.
+ *
+ * Keys returned by the iterator are not guaranteed to be in sorted order.
+ *
+ * @return an iterator over Key,Value pairs which meet the restrictions set on the scanner
+ */
++ @Override
+ public Iterator<Entry<Key,Value>> iterator();
+
+ /**
+ * This setting determines how long a scanner will automatically retry when a failure occurs. By default a scanner will retry forever.
+ *
+ * Setting to zero or Long.MAX_VALUE and TimeUnit.MILLISECONDS means to retry forever.
+ *
- * @param timeOut
+ * @param timeUnit
+ * determines how timeout is interpreted
+ * @since 1.5.0
+ */
+ public void setTimeout(long timeOut, TimeUnit timeUnit);
+
+ /**
+ * Returns the setting for how long a scanner will automatically retry when a failure occurs.
+ *
+ * @return the timeout configured for this scanner
+ * @since 1.5.0
+ */
+ public long getTimeout(TimeUnit timeUnit);
+
+ /**
+ * Closes any underlying connections on the scanner
+ * @since 1.5.0
+ */
+ public void close();
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
index 5197262,0000000..795ce9f
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
@@@ -1,320 -1,0 +1,318 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.impl.ConnectorImpl;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.OpTimer;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * <p>
+ * An implementation of instance that looks in zookeeper to find information needed to connect to an instance of accumulo.
+ *
+ * <p>
+ * The advantage of using zookeeper to obtain information about accumulo is that zookeeper is highly available, very responsive, and supports caching.
+ *
+ * <p>
+ * Because it is possible for multiple instances of accumulo to share a single set of zookeeper servers, all constructors require an accumulo instance name.
+ *
+ * If you do not know the instance names then run accumulo org.apache.accumulo.server.util.ListInstances on an accumulo server.
+ *
+ */
+
+public class ZooKeeperInstance implements Instance {
+
+ private static final Logger log = Logger.getLogger(ZooKeeperInstance.class);
+
+ private String instanceId = null;
+ private String instanceName = null;
+
+ private final ZooCache zooCache;
+
+ private final String zooKeepers;
+
+ private final int zooKeepersSessionTimeOut;
+
+ /**
+ *
+ * @param instanceName
+ * The name of specific accumulo instance. This is set at initialization time.
+ * @param zooKeepers
+ * A comma separated list of zoo keeper server locations. Each location can contain an optional port, of the format host:port.
+ */
+
+ public ZooKeeperInstance(String instanceName, String zooKeepers) {
+ this(instanceName, zooKeepers, (int) AccumuloConfiguration.getDefaultConfiguration().getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT));
+ }
+
+ /**
+ *
+ * @param instanceName
+ * The name of specific accumulo instance. This is set at initialization time.
+ * @param zooKeepers
+ * A comma separated list of zoo keeper server locations. Each location can contain an optional port, of the format host:port.
+ * @param sessionTimeout
+ * zoo keeper session time out in milliseconds.
+ */
+
+ public ZooKeeperInstance(String instanceName, String zooKeepers, int sessionTimeout) {
+ ArgumentChecker.notNull(instanceName, zooKeepers);
+ this.instanceName = instanceName;
+ this.zooKeepers = zooKeepers;
+ this.zooKeepersSessionTimeOut = sessionTimeout;
+ zooCache = ZooCache.getInstance(zooKeepers, sessionTimeout);
+ getInstanceID();
+ }
+
+ /**
+ *
+ * @param instanceId
+ * The UUID that identifies the accumulo instance you want to connect to.
+ * @param zooKeepers
+ * A comma separated list of zoo keeper server locations. Each location can contain an optional port, of the format host:port.
+ */
+
+ public ZooKeeperInstance(UUID instanceId, String zooKeepers) {
+ this(instanceId, zooKeepers, (int) AccumuloConfiguration.getDefaultConfiguration().getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT));
+ }
+
+ /**
+ *
+ * @param instanceId
+ * The UUID that identifies the accumulo instance you want to connect to.
+ * @param zooKeepers
+ * A comma separated list of zoo keeper server locations. Each location can contain an optional port, of the format host:port.
+ * @param sessionTimeout
+ * zoo keeper session time out in milliseconds.
+ */
+
+ public ZooKeeperInstance(UUID instanceId, String zooKeepers, int sessionTimeout) {
+ ArgumentChecker.notNull(instanceId, zooKeepers);
+ this.instanceId = instanceId.toString();
+ this.zooKeepers = zooKeepers;
+ this.zooKeepersSessionTimeOut = sessionTimeout;
+ zooCache = ZooCache.getInstance(zooKeepers, sessionTimeout);
+ }
+
+ @Override
+ public String getInstanceID() {
+ if (instanceId == null) {
+ // want the instance id to be stable for the life of this instance object,
+ // so only get it once
+ String instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + instanceName;
+ byte[] iidb = zooCache.get(instanceNamePath);
+ if (iidb == null) {
+ throw new RuntimeException("Instance name " + instanceName
+ + " does not exist in zookeeper. Run \"accumulo org.apache.accumulo.server.util.ListInstances\" to see a list.");
+ }
+ instanceId = new String(iidb, Constants.UTF8);
+ }
+
+ if (zooCache.get(Constants.ZROOT + "/" + instanceId) == null) {
+ if (instanceName == null)
+ throw new RuntimeException("Instance id " + instanceId + " does not exist in zookeeper");
+ throw new RuntimeException("Instance id " + instanceId + " pointed to by the name " + instanceName + " does not exist in zookeeper");
+ }
+
+ return instanceId;
+ }
+
+ @Override
+ public List<String> getMasterLocations() {
+ String masterLocPath = ZooUtil.getRoot(this) + Constants.ZMASTER_LOCK;
+
+ OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Looking up master location in zoocache.");
+ byte[] loc = ZooUtil.getLockData(zooCache, masterLocPath);
+ opTimer.stop("Found master at " + (loc == null ? null : new String(loc, Constants.UTF8)) + " in %DURATION%");
+
+ if (loc == null) {
+ return Collections.emptyList();
+ }
+
+ return Collections.singletonList(new String(loc, Constants.UTF8));
+ }
+
+ @Override
+ public String getRootTabletLocation() {
+ String zRootLocPath = ZooUtil.getRoot(this) + Constants.ZROOT_TABLET_LOCATION;
+
+ OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Looking up root tablet location in zookeeper.");
+ byte[] loc = zooCache.get(zRootLocPath);
+ opTimer.stop("Found root tablet at " + (loc == null ? null : new String(loc, Constants.UTF8)) + " in %DURATION%");
+
+ if (loc == null) {
+ return null;
+ }
+
+ return new String(loc, Constants.UTF8).split("\\|")[0];
+ }
+
+ @Override
+ public String getInstanceName() {
+ if (instanceName == null)
+ instanceName = lookupInstanceName(zooCache, UUID.fromString(getInstanceID()));
+
+ return instanceName;
+ }
+
+ @Override
+ public String getZooKeepers() {
+ return zooKeepers;
+ }
+
+ @Override
+ public int getZooKeepersSessionTimeOut() {
+ return zooKeepersSessionTimeOut;
+ }
+
+ @Override
+ @Deprecated
+ public Connector getConnector(String user, CharSequence pass) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(user, TextUtil.getBytes(new Text(pass.toString())));
+ }
+
+ @Override
+ @Deprecated
+ public Connector getConnector(String user, ByteBuffer pass) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(user, ByteBufferUtil.toBytes(pass));
+ }
+
+ @Override
+ public Connector getConnector(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(CredentialHelper.create(principal, token, getInstanceID()));
+ }
+
+ @SuppressWarnings("deprecation")
+ private Connector getConnector(TCredentials credential) throws AccumuloException, AccumuloSecurityException {
+ return new ConnectorImpl(this, credential);
+ }
+
+ @Override
+ @Deprecated
+ public Connector getConnector(String principal, byte[] pass) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(principal, new PasswordToken(pass));
+ }
+
+ private AccumuloConfiguration conf = null;
+
+ @Override
+ public AccumuloConfiguration getConfiguration() {
+ if (conf == null)
+ conf = AccumuloConfiguration.getDefaultConfiguration();
+ return conf;
+ }
+
+ @Override
+ public void setConfiguration(AccumuloConfiguration conf) {
+ this.conf = conf;
+ }
+
+ /**
+ * @deprecated Use {@link #lookupInstanceName(org.apache.accumulo.fate.zookeeper.ZooCache, UUID)} instead
+ */
+ @Deprecated
+ public static String lookupInstanceName(org.apache.accumulo.core.zookeeper.ZooCache zooCache, UUID instanceId) {
+ return lookupInstanceName((ZooCache) zooCache, instanceId);
+ }
+
+ /**
+ * Given a zooCache and instanceId, look up the instance name.
+ *
- * @param zooCache
- * @param instanceId
+ * @return the instance name
+ */
+ public static String lookupInstanceName(ZooCache zooCache, UUID instanceId) {
+ ArgumentChecker.notNull(zooCache, instanceId);
+ for (String name : zooCache.getChildren(Constants.ZROOT + Constants.ZINSTANCES)) {
+ String instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + name;
+ byte[] bytes = zooCache.get(instanceNamePath);
+ UUID iid = UUID.fromString(new String(bytes, Constants.UTF8));
+ if (iid.equals(instanceId)) {
+ return name;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * To be moved to server code. Only lives here to support certain client side utilities to minimize command-line options.
+ */
+ @Deprecated
+ public static String getInstanceIDFromHdfs(Path instanceDirectory) {
+ try {
+ FileSystem fs = FileUtil.getFileSystem(CachedConfiguration.getInstance(), AccumuloConfiguration.getSiteConfiguration());
+ FileStatus[] files = null;
+ try {
+ files = fs.listStatus(instanceDirectory);
+ } catch (FileNotFoundException ex) {
+ // ignored
+ }
+ log.debug("Trying to read instance id from " + instanceDirectory);
+ if (files == null || files.length == 0) {
+ log.error("unable obtain instance id at " + instanceDirectory);
+ throw new RuntimeException("Accumulo not initialized, there is no instance id at " + instanceDirectory);
+ } else if (files.length != 1) {
+ log.error("multiple potential instances in " + instanceDirectory);
+ throw new RuntimeException("Accumulo found multiple possible instance ids in " + instanceDirectory);
+ } else {
+ String result = files[0].getPath().getName();
+ return result;
+ }
+ } catch (IOException e) {
+ log.error("Problem reading instance id out of hdfs at " + instanceDirectory, e);
+ throw new RuntimeException("Can't tell if Accumulo is initialized; can't read instance id at " + instanceDirectory, e);
+ } catch (IllegalArgumentException exception) {
+ /* HDFS throws this when there's a UnknownHostException due to DNS troubles. */
+ if (exception.getCause() instanceof UnknownHostException) {
+ log.error("Problem reading instance id out of hdfs at " + instanceDirectory, exception);
+ }
+ throw exception;
+ }
+ }
+
+ @Deprecated
+ @Override
+ public Connector getConnector(org.apache.accumulo.core.security.thrift.AuthInfo auth) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(auth.user, auth.password);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/admin/ActiveCompaction.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/admin/ActiveCompaction.java
index f2876b2,0000000..9c39ea6
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/ActiveCompaction.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/ActiveCompaction.java
@@@ -1,185 -1,0 +1,184 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.admin;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+
+
+/**
+ *
+ * @since 1.5.0
+ */
+public class ActiveCompaction {
+
+ private org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction tac;
+ private Instance instance;
+
+ ActiveCompaction(Instance instance, org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction tac) {
+ this.tac = tac;
+ this.instance = instance;
+ }
+
+ public static enum CompactionType {
+ /**
+ * compaction to flush a tablets memory
+ */
+ MINOR,
+ /**
+ * compaction to flush a tablets memory and merge it with the tablets smallest file. This type compaction is done when a tablet has too many files
+ */
+ MERGE,
+ /**
+ * compaction that merges a subset of a tablets files into one file
+ */
+ MAJOR,
+ /**
+ * compaction that merges all of a tablets files into one file
+ */
+ FULL
+ };
+
+ public static enum CompactionReason {
+ /**
+ * compaction initiated by user
+ */
+ USER,
+ /**
+ * Compaction initiated by system
+ */
+ SYSTEM,
+ /**
+ * Compaction initiated by merge operation
+ */
+ CHOP,
+ /**
+ * idle compaction
+ */
+ IDLE,
+ /**
+ * Compaction initiated to close a unload a tablet
+ */
+ CLOSE
+ };
+
+ /**
+ *
+ * @return name of the table the compaction is running against
- * @throws TableNotFoundException
+ */
+
+ public String getTable() throws TableNotFoundException {
+ return Tables.getTableName(instance, getExtent().getTableId().toString());
+ }
+
+ /**
+ * @return tablet thats is compacting
+ */
+
+ public KeyExtent getExtent() {
+ return new KeyExtent(tac.getExtent());
+ }
+
+ /**
+ * @return how long the compaction has been running in milliseconds
+ */
+
+ public long getAge() {
+ return tac.getAge();
+ }
+
+ /**
+ * @return the files the compaction is reading from
+ */
+
+ public List<String> getInputFiles() {
+ return tac.getInputFiles();
+ }
+
+ /**
+ * @return file compactions is writing too
+ */
+
+ public String getOutputFile() {
+ return tac.getOutputFile();
+ }
+
+ /**
+ * @return the type of compaction
+ */
+ public CompactionType getType() {
+ return CompactionType.valueOf(tac.getType().name());
+ }
+
+ /**
+ * @return the reason the compaction was started
+ */
+
+ public CompactionReason getReason() {
+ return CompactionReason.valueOf(tac.getReason().name());
+ }
+
+ /**
+ * @return the locality group that is compacting
+ */
+
+ public String getLocalityGroup() {
+ return tac.getLocalityGroup();
+ }
+
+ /**
+ * @return the number of key/values read by the compaction
+ */
+
+ public long getEntriesRead() {
+ return tac.getEntriesRead();
+ }
+
+ /**
+ * @return the number of key/values written by the compaction
+ */
+
+ public long getEntriesWritten() {
+ return tac.getEntriesWritten();
+ }
+
+ /**
+ * @return the per compaction iterators configured
+ */
+
+ public List<IteratorSetting> getIterators() {
+ ArrayList<IteratorSetting> ret = new ArrayList<IteratorSetting>();
+
+ for (IterInfo ii : tac.getSsiList()) {
+ IteratorSetting settings = new IteratorSetting(ii.getPriority(), ii.getIterName(), ii.getClassName());
+ Map<String,String> options = tac.getSsio().get(ii.getIterName());
+ settings.addOptions(options);
+
+ ret.add(settings);
+ }
+
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
index fce0716,0000000..29ff2a6
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
@@@ -1,131 -1,0 +1,119 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.admin;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+
+/**
+ *
+ */
+public interface InstanceOperations {
+
+ /**
+ * Sets an system property in zookeeper. Tablet servers will pull this setting and override the equivalent setting in accumulo-site.xml. Changes can be seen
+ * using {@link #getSystemConfiguration()}
+ *
+ * @param property
+ * the name of a per-table property
+ * @param value
+ * the value to set a per-table property to
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ public void setProperty(final String property, final String value) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Removes a system property from zookeeper. Changes can be seen using {@link #getSystemConfiguration()}
+ *
+ * @param property
+ * the name of a per-table property
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ public void removeProperty(final String property) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ *
+ * @return A map of system properties set in zookeeper. If a property is not set in zookeeper, then it will return the value set in accumulo-site.xml on some
+ * server. If nothing is set in an accumulo-site.xml file it will return the default value for each property.
- * @throws AccumuloException
- * @throws AccumuloSecurityException
+ */
+
+ public Map<String,String> getSystemConfiguration() throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ *
+ * @return A map of system properties set in accumulo-site.xml on some server. If nothing is set in an accumulo-site.xml file it will return the default value
+ * for each property.
- * @throws AccumuloException
- * @throws AccumuloSecurityException
+ */
+
+ public Map<String,String> getSiteConfiguration() throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * List the currently active tablet servers participating in the accumulo instance
+ *
+ * @return A list of currently active tablet servers.
+ */
+
+ public List<String> getTabletServers();
+
+ /**
+ * List the active scans on tablet server.
+ *
+ * @param tserver
+ * The tablet server address should be of the form <ip address>:<port>
+ * @return A list of active scans on tablet server.
- * @throws AccumuloException
- * @throws AccumuloSecurityException
+ */
+
+ public List<ActiveScan> getActiveScans(String tserver) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * List the active compaction running on a tablet server
+ *
+ * @param tserver
+ * The tablet server address should be of the form <ip address>:<port>
+ * @return the list of active compactions
- * @throws AccumuloException
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+
+ public List<ActiveCompaction> getActiveCompactions(String tserver) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Throws an exception if a tablet server can not be contacted.
+ *
+ * @param tserver
+ * The tablet server address should be of the form <ip address>:<port>
- * @throws AccumuloException
+ * @since 1.5.0
+ */
+ public void ping(String tserver) throws AccumuloException;
+
+ /**
+ * Test to see if the instance can load the given class as the given type. This check does not consider per table classpaths, see
+ * {@link TableOperations#testClassLoad(String, String, String)}
+ *
- * @param className
- * @param asTypeName
+ * @return true if the instance can load the given class as the given type, false otherwise
- * @throws AccumuloException
+ */
+ public boolean testClassLoad(final String className, final String asTypeName) throws AccumuloException, AccumuloSecurityException;
+
+}
[43/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFileDumper.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFileDumper.java
index 9b9cd51,0000000..4b3e41c
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFileDumper.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFileDumper.java
@@@ -1,259 -1,0 +1,258 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.accumulo.core.file.rfile.bcfile;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.file.rfile.bcfile.BCFile.BlockRegion;
+import org.apache.accumulo.core.file.rfile.bcfile.BCFile.MetaIndexEntry;
+import org.apache.accumulo.core.file.rfile.bcfile.TFile.TFileIndexEntry;
+import org.apache.accumulo.core.file.rfile.bcfile.Utils.Version;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+
+/**
+ * Dumping the information of a TFile.
+ */
+class TFileDumper {
+ static final Log LOG = LogFactory.getLog(TFileDumper.class);
+
+ private TFileDumper() {
+ // namespace object not constructable.
+ }
+
+ private enum Align {
+ LEFT, CENTER, RIGHT, ZERO_PADDED;
+ static String format(String s, int width, Align align) {
+ if (s.length() >= width)
+ return s;
+ int room = width - s.length();
+ Align alignAdjusted = align;
+ if (room == 1) {
+ alignAdjusted = LEFT;
+ }
+ if (alignAdjusted == LEFT) {
+ return s + String.format("%" + room + "s", "");
+ }
+ if (alignAdjusted == RIGHT) {
+ return String.format("%" + room + "s", "") + s;
+ }
+ if (alignAdjusted == CENTER) {
+ int half = room / 2;
+ return String.format("%" + half + "s", "") + s + String.format("%" + (room - half) + "s", "");
+ }
+ throw new IllegalArgumentException("Unsupported alignment");
+ }
+
+ static String format(long l, int width, Align align) {
+ if (align == ZERO_PADDED) {
+ return String.format("%0" + width + "d", l);
+ }
+ return format(Long.toString(l), width, align);
+ }
+
+ static int calculateWidth(String caption, long max) {
+ return Math.max(caption.length(), Long.toString(max).length());
+ }
+ }
+
+ /**
+ * Dump information about TFile.
+ *
+ * @param file
+ * Path string of the TFile
+ * @param out
+ * PrintStream to output the information.
+ * @param conf
+ * The configuration object.
- * @throws IOException
+ */
+ static public void dumpInfo(String file, PrintStream out, Configuration conf) throws IOException {
+ final int maxKeySampleLen = 16;
+ Path path = new Path(file);
+ FileSystem fs = path.getFileSystem(conf);
+ long length = fs.getFileStatus(path).getLen();
+ FSDataInputStream fsdis = fs.open(path);
+ TFile.Reader reader = new TFile.Reader(fsdis, length, conf);
+ try {
+ LinkedHashMap<String,String> properties = new LinkedHashMap<String,String>();
+ int blockCnt = reader.readerBCF.getBlockCount();
+ int metaBlkCnt = reader.readerBCF.metaIndex.index.size();
+ properties.put("BCFile Version", reader.readerBCF.version.toString());
+ properties.put("TFile Version", reader.tfileMeta.version.toString());
+ properties.put("File Length", Long.toString(length));
+ properties.put("Data Compression", reader.readerBCF.getDefaultCompressionName());
+ properties.put("Record Count", Long.toString(reader.getEntryCount()));
+ properties.put("Sorted", Boolean.toString(reader.isSorted()));
+ if (reader.isSorted()) {
+ properties.put("Comparator", reader.getComparatorName());
+ }
+ properties.put("Data Block Count", Integer.toString(blockCnt));
+ long dataSize = 0, dataSizeUncompressed = 0;
+ if (blockCnt > 0) {
+ for (int i = 0; i < blockCnt; ++i) {
+ BlockRegion region = reader.readerBCF.dataIndex.getBlockRegionList().get(i);
+ dataSize += region.getCompressedSize();
+ dataSizeUncompressed += region.getRawSize();
+ }
+ properties.put("Data Block Bytes", Long.toString(dataSize));
+ if (reader.readerBCF.getDefaultCompressionName() != "none") {
+ properties.put("Data Block Uncompressed Bytes", Long.toString(dataSizeUncompressed));
+ properties.put("Data Block Compression Ratio", String.format("1:%.1f", (double) dataSizeUncompressed / dataSize));
+ }
+ }
+
+ properties.put("Meta Block Count", Integer.toString(metaBlkCnt));
+ long metaSize = 0, metaSizeUncompressed = 0;
+ if (metaBlkCnt > 0) {
+ Collection<MetaIndexEntry> metaBlks = reader.readerBCF.metaIndex.index.values();
+ boolean calculateCompression = false;
+ for (Iterator<MetaIndexEntry> it = metaBlks.iterator(); it.hasNext();) {
+ MetaIndexEntry e = it.next();
+ metaSize += e.getRegion().getCompressedSize();
+ metaSizeUncompressed += e.getRegion().getRawSize();
+ if (e.getCompressionAlgorithm() != Compression.Algorithm.NONE) {
+ calculateCompression = true;
+ }
+ }
+ properties.put("Meta Block Bytes", Long.toString(metaSize));
+ if (calculateCompression) {
+ properties.put("Meta Block Uncompressed Bytes", Long.toString(metaSizeUncompressed));
+ properties.put("Meta Block Compression Ratio", String.format("1:%.1f", (double) metaSizeUncompressed / metaSize));
+ }
+ }
+ properties.put("Meta-Data Size Ratio", String.format("1:%.1f", (double) dataSize / metaSize));
+ long leftOverBytes = length - dataSize - metaSize;
+ long miscSize = BCFile.Magic.size() * 2 + Long.SIZE / Byte.SIZE + Version.size();
+ long metaIndexSize = leftOverBytes - miscSize;
+ properties.put("Meta Block Index Bytes", Long.toString(metaIndexSize));
+ properties.put("Headers Etc Bytes", Long.toString(miscSize));
+ // Now output the properties table.
+ int maxKeyLength = 0;
+ Set<Map.Entry<String,String>> entrySet = properties.entrySet();
+ for (Iterator<Map.Entry<String,String>> it = entrySet.iterator(); it.hasNext();) {
+ Map.Entry<String,String> e = it.next();
+ if (e.getKey().length() > maxKeyLength) {
+ maxKeyLength = e.getKey().length();
+ }
+ }
+ for (Iterator<Map.Entry<String,String>> it = entrySet.iterator(); it.hasNext();) {
+ Map.Entry<String,String> e = it.next();
+ out.printf("%s : %s%n", Align.format(e.getKey(), maxKeyLength, Align.LEFT), e.getValue());
+ }
+ out.println();
+ reader.checkTFileDataIndex();
+ if (blockCnt > 0) {
+ String blkID = "Data-Block";
+ int blkIDWidth = Align.calculateWidth(blkID, blockCnt);
+ int blkIDWidth2 = Align.calculateWidth("", blockCnt);
+ String offset = "Offset";
+ int offsetWidth = Align.calculateWidth(offset, length);
+ String blkLen = "Length";
+ int blkLenWidth = Align.calculateWidth(blkLen, dataSize / blockCnt * 10);
+ String rawSize = "Raw-Size";
+ int rawSizeWidth = Align.calculateWidth(rawSize, dataSizeUncompressed / blockCnt * 10);
+ String records = "Records";
+ int recordsWidth = Align.calculateWidth(records, reader.getEntryCount() / blockCnt * 10);
+ String endKey = "End-Key";
+ int endKeyWidth = Math.max(endKey.length(), maxKeySampleLen * 2 + 5);
+
+ out.printf("%s %s %s %s %s %s%n", Align.format(blkID, blkIDWidth, Align.CENTER), Align.format(offset, offsetWidth, Align.CENTER),
+ Align.format(blkLen, blkLenWidth, Align.CENTER), Align.format(rawSize, rawSizeWidth, Align.CENTER),
+ Align.format(records, recordsWidth, Align.CENTER), Align.format(endKey, endKeyWidth, Align.LEFT));
+
+ for (int i = 0; i < blockCnt; ++i) {
+ BlockRegion region = reader.readerBCF.dataIndex.getBlockRegionList().get(i);
+ TFileIndexEntry indexEntry = reader.tfileIndex.getEntry(i);
+ out.printf("%s %s %s %s %s ", Align.format(Align.format(i, blkIDWidth2, Align.ZERO_PADDED), blkIDWidth, Align.LEFT),
+ Align.format(region.getOffset(), offsetWidth, Align.LEFT), Align.format(region.getCompressedSize(), blkLenWidth, Align.LEFT),
+ Align.format(region.getRawSize(), rawSizeWidth, Align.LEFT), Align.format(indexEntry.kvEntries, recordsWidth, Align.LEFT));
+ byte[] key = indexEntry.key;
+ boolean asAscii = true;
+ int sampleLen = Math.min(maxKeySampleLen, key.length);
+ for (int j = 0; j < sampleLen; ++j) {
+ byte b = key[j];
+ if ((b < 32 && b != 9) || (b == 127)) {
+ asAscii = false;
+ }
+ }
+ if (!asAscii) {
+ out.print("0X");
+ for (int j = 0; j < sampleLen; ++j) {
+ byte b = key[i];
+ out.printf("%X", b);
+ }
+ } else {
+ out.print(new String(key, 0, sampleLen, Constants.UTF8));
+ }
+ if (sampleLen < key.length) {
+ out.print("...");
+ }
+ out.println();
+ }
+ }
+
+ out.println();
+ if (metaBlkCnt > 0) {
+ String name = "Meta-Block";
+ int maxNameLen = 0;
+ Set<Map.Entry<String,MetaIndexEntry>> metaBlkEntrySet = reader.readerBCF.metaIndex.index.entrySet();
+ for (Iterator<Map.Entry<String,MetaIndexEntry>> it = metaBlkEntrySet.iterator(); it.hasNext();) {
+ Map.Entry<String,MetaIndexEntry> e = it.next();
+ if (e.getKey().length() > maxNameLen) {
+ maxNameLen = e.getKey().length();
+ }
+ }
+ int nameWidth = Math.max(name.length(), maxNameLen);
+ String offset = "Offset";
+ int offsetWidth = Align.calculateWidth(offset, length);
+ String blkLen = "Length";
+ int blkLenWidth = Align.calculateWidth(blkLen, metaSize / metaBlkCnt * 10);
+ String rawSize = "Raw-Size";
+ int rawSizeWidth = Align.calculateWidth(rawSize, metaSizeUncompressed / metaBlkCnt * 10);
+ String compression = "Compression";
+ int compressionWidth = compression.length();
+ out.printf("%s %s %s %s %s%n", Align.format(name, nameWidth, Align.CENTER), Align.format(offset, offsetWidth, Align.CENTER),
+ Align.format(blkLen, blkLenWidth, Align.CENTER), Align.format(rawSize, rawSizeWidth, Align.CENTER),
+ Align.format(compression, compressionWidth, Align.LEFT));
+
+ for (Iterator<Map.Entry<String,MetaIndexEntry>> it = metaBlkEntrySet.iterator(); it.hasNext();) {
+ Map.Entry<String,MetaIndexEntry> e = it.next();
+ String blkName = e.getValue().getMetaName();
+ BlockRegion region = e.getValue().getRegion();
+ String blkCompression = e.getValue().getCompressionAlgorithm().getName();
+ out.printf("%s %s %s %s %s%n", Align.format(blkName, nameWidth, Align.LEFT), Align.format(region.getOffset(), offsetWidth, Align.LEFT),
+ Align.format(region.getCompressedSize(), blkLenWidth, Align.LEFT), Align.format(region.getRawSize(), rawSizeWidth, Align.LEFT),
+ Align.format(blkCompression, compressionWidth, Align.LEFT));
+ }
+ }
+ } finally {
+ IOUtils.cleanup(LOG, reader, fsdis);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Utils.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Utils.java
index 45a59f6,0000000..9131d30
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Utils.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Utils.java
@@@ -1,485 -1,0 +1,474 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.accumulo.core.file.rfile.bcfile;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.hadoop.io.Text;
+
+/**
+ * Supporting Utility classes used by TFile, and shared by users of TFile.
+ */
+public final class Utils {
+
+ /**
+ * Prevent the instantiation of Utils.
+ */
+ private Utils() {
+ // nothing
+ }
+
+ /**
+ * Encoding an integer into a variable-length encoding format. Synonymous to <code>Utils#writeVLong(out, n)</code>.
+ *
+ * @param out
+ * output stream
+ * @param n
+ * The integer to be encoded
- * @throws IOException
+ * @see Utils#writeVLong(DataOutput, long)
+ */
+ public static void writeVInt(DataOutput out, int n) throws IOException {
+ writeVLong(out, n);
+ }
+
+ /**
+ * Encoding a Long integer into a variable-length encoding format.
+ * <ul>
+ * <li>if n in [-32, 127): encode in one byte with the actual value. Otherwise,
+ * <li>if n in [-20*2^8, 20*2^8): encode in two bytes: byte[0] = n/256 - 52; byte[1]=n&0xff. Otherwise,
+ * <li>if n IN [-16*2^16, 16*2^16): encode in three bytes: byte[0]=n/2^16 - 88; byte[1]=(n>>8)&0xff; byte[2]=n&0xff. Otherwise,
+ * <li>if n in [-8*2^24, 8*2^24): encode in four bytes: byte[0]=n/2^24 - 112; byte[1] = (n>>16)&0xff; byte[2] = (n>>8)&0xff; byte[3]=n&0xff. Otherwise:
+ * <li>if n in [-2^31, 2^31): encode in five bytes: byte[0]=-125; byte[1] = (n>>24)&0xff; byte[2]=(n>>16)&0xff; byte[3]=(n>>8)&0xff; byte[4]=n&0xff;
+ * <li>if n in [-2^39, 2^39): encode in six bytes: byte[0]=-124; byte[1] = (n>>32)&0xff; byte[2]=(n>>24)&0xff; byte[3]=(n>>16)&0xff; byte[4]=(n>>8)&0xff;
+ * byte[5]=n&0xff
+ * <li>if n in [-2^47, 2^47): encode in seven bytes: byte[0]=-123; byte[1] = (n>>40)&0xff; byte[2]=(n>>32)&0xff; byte[3]=(n>>24)&0xff; byte[4]=(n>>16)&0xff;
+ * byte[5]=(n>>8)&0xff; byte[6]=n&0xff;
+ * <li>if n in [-2^55, 2^55): encode in eight bytes: byte[0]=-122; byte[1] = (n>>48)&0xff; byte[2] = (n>>40)&0xff; byte[3]=(n>>32)&0xff; byte[4]=(n>>24)&0xff;
+ * byte[5]=(n>>16)&0xff; byte[6]=(n>>8)&0xff; byte[7]=n&0xff;
+ * <li>if n in [-2^63, 2^63): encode in nine bytes: byte[0]=-121; byte[1] = (n>>54)&0xff; byte[2] = (n>>48)&0xff; byte[3] = (n>>40)&0xff;
+ * byte[4]=(n>>32)&0xff; byte[5]=(n>>24)&0xff; byte[6]=(n>>16)&0xff; byte[7]=(n>>8)&0xff; byte[8]=n&0xff;
+ * </ul>
+ *
+ * @param out
+ * output stream
+ * @param n
+ * the integer number
- * @throws IOException
+ */
+ @SuppressWarnings("fallthrough")
+ public static void writeVLong(DataOutput out, long n) throws IOException {
+ if ((n < 128) && (n >= -32)) {
+ out.writeByte((int) n);
+ return;
+ }
+
+ long un = (n < 0) ? ~n : n;
+ // how many bytes do we need to represent the number with sign bit?
+ int len = (Long.SIZE - Long.numberOfLeadingZeros(un)) / 8 + 1;
+ int firstByte = (int) (n >> ((len - 1) * 8));
+ switch (len) {
+ case 1:
+ // fall it through to firstByte==-1, len=2.
+ firstByte >>= 8;
+ case 2:
+ if ((firstByte < 20) && (firstByte >= -20)) {
+ out.writeByte(firstByte - 52);
+ out.writeByte((int) n);
+ return;
+ }
+ // fall it through to firstByte==0/-1, len=3.
+ firstByte >>= 8;
+ case 3:
+ if ((firstByte < 16) && (firstByte >= -16)) {
+ out.writeByte(firstByte - 88);
+ out.writeShort((int) n);
+ return;
+ }
+ // fall it through to firstByte==0/-1, len=4.
+ firstByte >>= 8;
+ case 4:
+ if ((firstByte < 8) && (firstByte >= -8)) {
+ out.writeByte(firstByte - 112);
+ out.writeShort(((int) n) >>> 8);
+ out.writeByte((int) n);
+ return;
+ }
+ out.writeByte(len - 129);
+ out.writeInt((int) n);
+ return;
+ case 5:
+ out.writeByte(len - 129);
+ out.writeInt((int) (n >>> 8));
+ out.writeByte((int) n);
+ return;
+ case 6:
+ out.writeByte(len - 129);
+ out.writeInt((int) (n >>> 16));
+ out.writeShort((int) n);
+ return;
+ case 7:
+ out.writeByte(len - 129);
+ out.writeInt((int) (n >>> 24));
+ out.writeShort((int) (n >>> 8));
+ out.writeByte((int) n);
+ return;
+ case 8:
+ out.writeByte(len - 129);
+ out.writeLong(n);
+ return;
+ default:
+ throw new RuntimeException("Internel error");
+ }
+ }
+
+ /**
+ * Decoding the variable-length integer. Synonymous to <code>(int)Utils#readVLong(in)</code>.
+ *
+ * @param in
+ * input stream
+ * @return the decoded integer
- * @throws IOException
+ *
+ * @see Utils#readVLong(DataInput)
+ */
+ public static int readVInt(DataInput in) throws IOException {
+ long ret = readVLong(in);
+ if ((ret > Integer.MAX_VALUE) || (ret < Integer.MIN_VALUE)) {
+ throw new RuntimeException("Number too large to be represented as Integer");
+ }
+ return (int) ret;
+ }
+
+ /**
+ * Decoding the variable-length integer. Suppose the value of the first byte is FB, and the following bytes are NB[*].
+ * <ul>
+ * <li>if (FB >= -32), return (long)FB;
+ * <li>if (FB in [-72, -33]), return (FB+52)<<8 + NB[0]&0xff;
+ * <li>if (FB in [-104, -73]), return (FB+88)<<16 + (NB[0]&0xff)<<8 + NB[1]&0xff;
+ * <li>if (FB in [-120, -105]), return (FB+112)<<24 + (NB[0]&0xff)<<16 + (NB[1]&0xff)<<8 + NB[2]&0xff;
+ * <li>if (FB in [-128, -121]), return interpret NB[FB+129] as a signed big-endian integer.
+ *
+ * @param in
+ * input stream
+ * @return the decoded long integer.
- * @throws IOException
+ */
+
+ public static long readVLong(DataInput in) throws IOException {
+ int firstByte = in.readByte();
+ if (firstByte >= -32) {
+ return firstByte;
+ }
+
+ switch ((firstByte + 128) / 8) {
+ case 11:
+ case 10:
+ case 9:
+ case 8:
+ case 7:
+ return ((firstByte + 52) << 8) | in.readUnsignedByte();
+ case 6:
+ case 5:
+ case 4:
+ case 3:
+ return ((firstByte + 88) << 16) | in.readUnsignedShort();
+ case 2:
+ case 1:
+ return ((firstByte + 112) << 24) | (in.readUnsignedShort() << 8) | in.readUnsignedByte();
+ case 0:
+ int len = firstByte + 129;
+ switch (len) {
+ case 4:
+ return in.readInt();
+ case 5:
+ return ((long) in.readInt()) << 8 | in.readUnsignedByte();
+ case 6:
+ return ((long) in.readInt()) << 16 | in.readUnsignedShort();
+ case 7:
+ return ((long) in.readInt()) << 24 | (in.readUnsignedShort() << 8) | in.readUnsignedByte();
+ case 8:
+ return in.readLong();
+ default:
+ throw new IOException("Corrupted VLong encoding");
+ }
+ default:
+ throw new RuntimeException("Internal error");
+ }
+ }
+
+ /**
+ * Write a String as a VInt n, followed by n Bytes as in Text format.
- *
- * @param out
- * @param s
- * @throws IOException
+ */
+ public static void writeString(DataOutput out, String s) throws IOException {
+ if (s != null) {
+ Text text = new Text(s);
+ byte[] buffer = text.getBytes();
+ int len = text.getLength();
+ writeVInt(out, len);
+ out.write(buffer, 0, len);
+ } else {
+ writeVInt(out, -1);
+ }
+ }
+
+ /**
+ * Read a String as a VInt n, followed by n Bytes in Text format.
+ *
+ * @param in
+ * The input stream.
+ * @return The string
- * @throws IOException
+ */
+ public static String readString(DataInput in) throws IOException {
+ int length = readVInt(in);
+ if (length == -1)
+ return null;
+ byte[] buffer = new byte[length];
+ in.readFully(buffer);
+ return Text.decode(buffer);
+ }
+
+ /**
+ * A generic Version class. We suggest applications built on top of TFile use this class to maintain version information in their meta blocks.
+ *
+ * A version number consists of a major version and a minor version. The suggested usage of major and minor version number is to increment major version
+ * number when the new storage format is not backward compatible, and increment the minor version otherwise.
+ */
+ public static final class Version implements Comparable<Version> {
+ private final short major;
+ private final short minor;
+
+ /**
+ * Construct the Version object by reading from the input stream.
+ *
+ * @param in
+ * input stream
- * @throws IOException
+ */
+ public Version(DataInput in) throws IOException {
+ major = in.readShort();
+ minor = in.readShort();
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param major
+ * major version.
+ * @param minor
+ * minor version.
+ */
+ public Version(short major, short minor) {
+ this.major = major;
+ this.minor = minor;
+ }
+
+ /**
+ * Write the object to a DataOutput. The serialized format of the Version is major version followed by minor version, both as big-endian short integers.
+ *
+ * @param out
+ * The DataOutput object.
- * @throws IOException
+ */
+ public void write(DataOutput out) throws IOException {
+ out.writeShort(major);
+ out.writeShort(minor);
+ }
+
+ /**
+ * Get the major version.
+ *
+ * @return Major version.
+ */
+ public int getMajor() {
+ return major;
+ }
+
+ /**
+ * Get the minor version.
+ *
+ * @return The minor version.
+ */
+ public int getMinor() {
+ return minor;
+ }
+
+ /**
+ * Get the size of the serialized Version object.
+ *
+ * @return serialized size of the version object.
+ */
+ public static int size() {
+ return (Short.SIZE + Short.SIZE) / Byte.SIZE;
+ }
+
+ /**
+ * Return a string representation of the version.
+ */
+ @Override
+ public String toString() {
+ return new StringBuilder("v").append(major).append(".").append(minor).toString();
+ }
+
+ /**
+ * Test compatibility.
+ *
+ * @param other
+ * The Version object to test compatibility with.
+ * @return true if both versions have the same major version number; false otherwise.
+ */
+ public boolean compatibleWith(Version other) {
+ return major == other.major;
+ }
+
+ /**
+ * Compare this version with another version.
+ */
+ @Override
+ public int compareTo(Version that) {
+ if (major != that.major) {
+ return major - that.major;
+ }
+ return minor - that.minor;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other)
+ return true;
+ if (!(other instanceof Version))
+ return false;
+ return compareTo((Version) other) == 0;
+ }
+
+ @Override
+ public int hashCode() {
+ return (major << 16 + minor);
+ }
+ }
+
+ /**
+ * Lower bound binary search. Find the index to the first element in the list that compares greater than or equal to key.
+ *
+ * @param <T>
+ * Type of the input key.
+ * @param list
+ * The list
+ * @param key
+ * The input key.
+ * @param cmp
+ * Comparator for the key.
+ * @return The index to the desired element if it exists; or list.size() otherwise.
+ */
+ public static <T> int lowerBound(List<? extends T> list, T key, Comparator<? super T> cmp) {
+ int low = 0;
+ int high = list.size();
+
+ while (low < high) {
+ int mid = (low + high) >>> 1;
+ T midVal = list.get(mid);
+ int ret = cmp.compare(midVal, key);
+ if (ret < 0)
+ low = mid + 1;
+ else
+ high = mid;
+ }
+ return low;
+ }
+
+ /**
+ * Upper bound binary search. Find the index to the first element in the list that compares greater than the input key.
+ *
+ * @param <T>
+ * Type of the input key.
+ * @param list
+ * The list
+ * @param key
+ * The input key.
+ * @param cmp
+ * Comparator for the key.
+ * @return The index to the desired element if it exists; or list.size() otherwise.
+ */
+ public static <T> int upperBound(List<? extends T> list, T key, Comparator<? super T> cmp) {
+ int low = 0;
+ int high = list.size();
+
+ while (low < high) {
+ int mid = (low + high) >>> 1;
+ T midVal = list.get(mid);
+ int ret = cmp.compare(midVal, key);
+ if (ret <= 0)
+ low = mid + 1;
+ else
+ high = mid;
+ }
+ return low;
+ }
+
+ /**
+ * Lower bound binary search. Find the index to the first element in the list that compares greater than or equal to key.
+ *
+ * @param <T>
+ * Type of the input key.
+ * @param list
+ * The list
+ * @param key
+ * The input key.
+ * @return The index to the desired element if it exists; or list.size() otherwise.
+ */
+ public static <T> int lowerBound(List<? extends Comparable<? super T>> list, T key) {
+ int low = 0;
+ int high = list.size();
+
+ while (low < high) {
+ int mid = (low + high) >>> 1;
+ Comparable<? super T> midVal = list.get(mid);
+ int ret = midVal.compareTo(key);
+ if (ret < 0)
+ low = mid + 1;
+ else
+ high = mid;
+ }
+ return low;
+ }
+
+ /**
+ * Upper bound binary search. Find the index to the first element in the list that compares greater than the input key.
+ *
+ * @param <T>
+ * Type of the input key.
+ * @param list
+ * The list
+ * @param key
+ * The input key.
+ * @return The index to the desired element if it exists; or list.size() otherwise.
+ */
+ public static <T> int upperBound(List<? extends Comparable<? super T>> list, T key) {
+ int low = 0;
+ int high = list.size();
+
+ while (low < high) {
+ int mid = (low + high) >>> 1;
+ Comparable<? super T> midVal = list.get(mid);
+ int ret = midVal.compareTo(key);
+ if (ret <= 0)
+ low = mid + 1;
+ else
+ high = mid;
+ }
+ return low;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
index 54a1333,0000000..5b7b05c
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
@@@ -1,249 -1,0 +1,243 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+
+/**
+ * A Combiner that decodes each Value to type V before reducing, then encodes the result of typedReduce back to Value.
+ *
+ * Subclasses must implement a typedReduce method: public V typedReduce(Key key, Iterator<V> iter);
+ *
+ * This typedReduce method will be passed the most recent Key and an iterator over the Values (translated to Vs) for all non-deleted versions of that Key.
+ *
+ * Subclasses may implement a switch on the "type" variable to choose an Encoder in their init method.
+ */
+public abstract class TypedValueCombiner<V> extends Combiner {
+ private Encoder<V> encoder = null;
+ private boolean lossy = false;
+
+ protected static final String LOSSY = "lossy";
+
+ /**
+ * A Java Iterator that translates an Iterator<Value> to an Iterator<V> using the decode method of an Encoder.
+ */
+ private static class VIterator<V> implements Iterator<V> {
+ private Iterator<Value> source;
+ private Encoder<V> encoder;
+ private boolean lossy;
+
+ /**
+ * Constructs an Iterator<V> from an Iterator<Value>
+ *
+ * @param iter
+ * The source iterator
+ *
+ * @param encoder
+ * The Encoder whose decode method is used to translate from Value to V
+ *
+ * @param lossy
+ * Determines whether to error on failure to decode or ignore and move on
+ */
+ VIterator(Iterator<Value> iter, Encoder<V> encoder, boolean lossy) {
+ this.source = iter;
+ this.encoder = encoder;
+ this.lossy = lossy;
+ }
+
+ V next = null;
+ boolean hasNext = false;
+
+ @Override
+ public boolean hasNext() {
+ if (hasNext)
+ return true;
+
+ while (true) {
+ if (!source.hasNext())
+ return false;
+ try {
+ next = encoder.decode(source.next().get());
+ return hasNext = true;
+ } catch (ValueFormatException vfe) {
+ if (!lossy)
+ throw vfe;
+ }
+ }
+ }
+
+ @Override
+ public V next() {
+ if (!hasNext && !hasNext())
+ throw new NoSuchElementException();
+ V toRet = next;
+ next = null;
+ hasNext = false;
+ return toRet;
+ }
+
+ @Override
+ public void remove() {
+ source.remove();
+ }
+ }
+
+ /**
+ * An interface for translating from byte[] to V and back.
+ */
+ public static interface Encoder<V> {
+ public byte[] encode(V v);
+
+ public V decode(byte[] b) throws ValueFormatException;
+ }
+
+ /**
+ * Sets the Encoder<V> used to translate Values to V and back.
- *
- * @param encoder
+ */
+ protected void setEncoder(Encoder<V> encoder) {
+ this.encoder = encoder;
+ }
+
+ /**
+ * Instantiates and sets the Encoder<V> used to translate Values to V and back.
+ *
- * @param encoderClass
+ * @throws IllegalArgumentException
+ * if ClassNotFoundException, InstantiationException, or IllegalAccessException occurs
+ */
+ protected void setEncoder(String encoderClass) {
+ try {
+ @SuppressWarnings("unchecked")
+ Class<? extends Encoder<V>> clazz = (Class<? extends Encoder<V>>) AccumuloVFSClassLoader.loadClass(encoderClass, Encoder.class);
+ encoder = clazz.newInstance();
+ } catch (ClassNotFoundException e) {
+ throw new IllegalArgumentException(e);
+ } catch (InstantiationException e) {
+ throw new IllegalArgumentException(e);
+ } catch (IllegalAccessException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+
+ /**
+ * Tests whether v remains the same when encoded and decoded with the current encoder.
+ *
- * @param v
+ * @throws IllegalStateException
+ * if an encoder has not been set.
+ * @throws IllegalArgumentException
+ * if the test fails.
+ */
+ protected void testEncoder(V v) {
+ if (encoder == null)
+ throw new IllegalStateException("encoder has not been initialized");
+ testEncoder(encoder, v);
+ }
+
+ /**
+ * Tests whether v remains the same when encoded and decoded with the given encoder.
+ *
- * @param encoder
- * @param v
+ * @throws IllegalArgumentException
+ * if the test fails.
+ */
+ public static <V> void testEncoder(Encoder<V> encoder, V v) {
+ try {
+ if (!v.equals(encoder.decode(encoder.encode(v))))
+ throw new IllegalArgumentException("something wrong with " + encoder.getClass().getName() + " -- doesn't encode and decode " + v + " properly");
+ } catch (ClassCastException e) {
+ throw new IllegalArgumentException(encoder.getClass().getName() + " doesn't encode " + v.getClass().getName());
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ TypedValueCombiner<V> newInstance = (TypedValueCombiner<V>) super.deepCopy(env);
+ newInstance.setEncoder(encoder);
+ return newInstance;
+ }
+
+ @Override
+ public Value reduce(Key key, Iterator<Value> iter) {
+ return new Value(encoder.encode(typedReduce(key, new VIterator<V>(iter, encoder, lossy))));
+ }
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ super.init(source, options, env);
+ setLossyness(options);
+ }
+
+ private void setLossyness(Map<String,String> options) {
+ String loss = options.get(LOSSY);
+ if (loss == null)
+ lossy = false;
+ else
+ lossy = Boolean.parseBoolean(loss);
+ }
+
+ @Override
+ public IteratorOptions describeOptions() {
+ IteratorOptions io = super.describeOptions();
+ io.addNamedOption(LOSSY, "if true, failed decodes are ignored. Otherwise combiner will error on failed decodes (default false): <TRUE|FALSE>");
+ return io;
+ }
+
+ @Override
+ public boolean validateOptions(Map<String,String> options) {
+ if (super.validateOptions(options) == false)
+ return false;
+ try {
+ setLossyness(options);
+ } catch (Exception e) {
+ throw new IllegalArgumentException("bad boolean " + LOSSY + ":" + options.get(LOSSY));
+ }
+ return true;
+ }
+
+ /**
+ * A convenience method to set the "lossy" option on a TypedValueCombiner. If true, the combiner will ignore any values which fail to decode. Otherwise, the
+ * combiner will throw an error which will interrupt the action (and prevent potential data loss). False is the default behavior.
+ *
+ * @param is
+ * iterator settings object to configure
+ * @param lossy
+ * if true the combiner will ignored values which fail to decode; otherwise error.
+ */
+ public static void setLossyness(IteratorSetting is, boolean lossy) {
+ is.addOption(LOSSY, Boolean.toString(lossy));
+ }
+
+ /**
+ * Reduces a list of V into a single V.
+ *
+ * @param key
+ * The most recent version of the Key being reduced.
+ *
+ * @param iter
+ * An iterator over the V for different versions of the key.
+ *
+ * @return The combined V.
+ */
+ public abstract V typedReduce(Key key, Iterator<V> iter);
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/ValueFormatException.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/ValueFormatException.java
index 7bb2228,0000000..7ede7fe
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/ValueFormatException.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/ValueFormatException.java
@@@ -1,40 -1,0 +1,34 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators;
+
+/**
+ * Exception used for TypedValueCombiner and it's Encoders decode() function
+ */
+public class ValueFormatException extends IllegalArgumentException {
+
- /**
- * @param string
- */
+ public ValueFormatException(String string) {
+ super(string);
+ }
+
- /**
- * @param nfe
- */
+ public ValueFormatException(Exception nfe) {
+ super(nfe);
+ }
+
+ private static final long serialVersionUID = 4170291568272971821L;
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/system/MapFileIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/system/MapFileIterator.java
index e5fe62a,0000000..37a234c
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/system/MapFileIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/system/MapFileIterator.java
@@@ -1,162 -1,0 +1,156 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators.system;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileSKVIterator;
+import org.apache.accumulo.core.file.NoSuchMetaStoreException;
+import org.apache.accumulo.core.file.map.MapFileUtil;
+import org.apache.accumulo.core.iterators.IterationInterruptedException;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.MapFile.Reader;
+import org.apache.log4j.Logger;
+
+public class MapFileIterator implements FileSKVIterator {
+ private static final Logger log = Logger.getLogger(MapFileIterator.class);
+
+ private Reader reader;
+ private Value topValue;
+ private Key topKey;
+ private AtomicBoolean interruptFlag;
+ private int interruptCheckCount = 0;
+ private FileSystem fs;
+ private String dirName;
+
- /**
- * @param acuconf
- * @param fs
- * @param dir
- * @param conf
- * @throws IOException
- */
+ public MapFileIterator(AccumuloConfiguration acuconf, FileSystem fs, String dir, Configuration conf) throws IOException {
+ this.reader = MapFileUtil.openMapFile(acuconf, fs, dir, conf);
+ this.fs = fs;
+ this.dirName = dir;
+ }
+
+ @Override
+ public void setInterruptFlag(AtomicBoolean flag) {
+ this.interruptFlag = flag;
+ }
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean hasTop() {
+ return topKey != null;
+ }
+
+ @Override
+ public void next() throws IOException {
+ if (interruptFlag != null && interruptCheckCount++ % 100 == 0 && interruptFlag.get())
+ throw new IterationInterruptedException();
+
+ reader.next(topKey, topValue);
+ }
+
++ @Override
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
+ if (columnFamilies.size() != 0 || inclusive) {
+ throw new IllegalArgumentException("I do not know how to filter column families");
+ }
+
+ if (range == null)
+ throw new IllegalArgumentException("Cannot seek to null range");
+
+ if (interruptFlag != null && interruptFlag.get())
+ throw new IterationInterruptedException();
+
+ Key key = range.getStartKey();
+ if (key == null) {
+ key = new Key();
+ }
+
+ reader.seek(key);
+
+ while (hasTop() && range.beforeStartKey(getTopKey())) {
+ next();
+ }
+ }
+
+ @Override
+ public Key getTopKey() {
+ return topKey;
+ }
+
+ @Override
+ public Value getTopValue() {
+ return topValue;
+ }
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ try {
+ SortedKeyValueIterator<Key,Value> other = env.reserveMapFileReader(dirName);
+ ((InterruptibleIterator) other).setInterruptFlag(interruptFlag);
+ log.debug("deep copying MapFile: " + this + " -> " + other);
+ return other;
+ } catch (IOException e) {
+ log.error("failed to clone map file reader", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public Key getFirstKey() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Key getLastKey() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DataInputStream getMetaStore(String name) throws IOException {
+ Path path = new Path(this.dirName, name);
+ if (!fs.exists(path))
+ throw new NoSuchMetaStoreException("name = " + name);
+ return fs.open(path);
+ }
+
+ @Override
+ public void closeDeepCopies() throws IOException {
+ // nothing to do, deep copies are externally managed/closed
+ }
+
+ @Override
+ public void close() throws IOException {
+ reader.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
index 4f8207c,0000000..86798dd
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
@@@ -1,104 -1,0 +1,101 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators.user;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Map;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.Filter;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+
+/**
+ * This iterator provides exact string matching. It searches both the Key and Value for the string. The string to match is specified by the "term" option.
+ */
+public class GrepIterator extends Filter {
+
+ private byte term[];
+
+ @Override
+ public boolean accept(Key k, Value v) {
+ return match(v.get()) || match(k.getRowData()) || match(k.getColumnFamilyData()) || match(k.getColumnQualifierData());
+ }
+
+ private boolean match(ByteSequence bs) {
+ return indexOf(bs.getBackingArray(), bs.offset(), bs.length(), term) >= 0;
+ }
+
+ private boolean match(byte[] ba) {
+ return indexOf(ba, 0, ba.length, term) >= 0;
+ }
+
+ // copied code below from java string and modified
+
+ private static int indexOf(byte[] source, int sourceOffset, int sourceCount, byte[] target) {
+ byte first = target[0];
+ int targetCount = target.length;
+ int max = sourceOffset + (sourceCount - targetCount);
+
+ for (int i = sourceOffset; i <= max; i++) {
+ /* Look for first character. */
+ if (source[i] != first) {
+ while (++i <= max && source[i] != first)
+ continue;
+ }
+
+ /* Found first character, now look at the rest of v2 */
+ if (i <= max) {
+ int j = i + 1;
+ int end = j + targetCount - 1;
+ for (int k = 1; j < end && source[j] == target[k]; j++, k++)
+ continue;
+
+ if (j == end) {
+ /* Found whole string. */
+ return i - sourceOffset;
+ }
+ }
+ }
+ return -1;
+ }
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ GrepIterator copy = (GrepIterator) super.deepCopy(env);
+ copy.term = Arrays.copyOf(term, term.length);
+ return copy;
+ }
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ super.init(source, options, env);
+ term = options.get("term").getBytes(Constants.UTF8);
+ }
+
+ /**
+ * Encode the grep term as an option for a ScanIterator
- *
- * @param cfg
- * @param term
+ */
+ public static void setTerm(IteratorSetting cfg, String term) {
+ cfg.addOption("term", term);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
index 447200b,0000000..39cba6d
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
@@@ -1,558 -1,0 +1,548 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators.user;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+/**
+ * This iterator facilitates document-partitioned indexing. It involves grouping a set of documents together and indexing those documents into a single row of
+ * an Accumulo table. This allows a tablet server to perform boolean AND operations on terms in the index.
+ *
+ * The table structure should have the following form:
+ *
+ * row: shardID, colfam: term, colqual: docID
+ *
+ * When you configure this iterator with a set of terms (column families), it will return only the docIDs that appear with all of the specified terms. The
+ * result will have an empty column family, as follows:
+ *
+ * row: shardID, colfam: (empty), colqual: docID
+ *
+ * This iterator is commonly used with BatchScanner or AccumuloInputFormat, to parallelize the search over all shardIDs.
+ *
+ * This iterator will *ignore* any columnFamilies passed to {@link #seek(Range, Collection, boolean)} as it performs intersections over terms. Extending classes
+ * should override the {@link TermSource#seekColfams} in their implementation's {@link #init(SortedKeyValueIterator, Map, IteratorEnvironment)} method.
+ *
+ * README.shard in docs/examples shows an example of using the IntersectingIterator.
+ */
+public class IntersectingIterator implements SortedKeyValueIterator<Key,Value> {
+
+ protected Text nullText = new Text();
+
+ protected Text getPartition(Key key) {
+ return key.getRow();
+ }
+
+ protected Text getTerm(Key key) {
+ return key.getColumnFamily();
+ }
+
+ protected Text getDocID(Key key) {
+ return key.getColumnQualifier();
+ }
+
+ protected Key buildKey(Text partition, Text term) {
+ return new Key(partition, (term == null) ? nullText : term);
+ }
+
+ protected Key buildKey(Text partition, Text term, Text docID) {
+ return new Key(partition, (term == null) ? nullText : term, docID);
+ }
+
+ protected Key buildFollowingPartitionKey(Key key) {
+ return key.followingKey(PartialKey.ROW);
+ }
+
+ protected static final Logger log = Logger.getLogger(IntersectingIterator.class);
+
+ public static class TermSource {
+ public SortedKeyValueIterator<Key,Value> iter;
+ public Text term;
+ public Collection<ByteSequence> seekColfams;
+ public boolean notFlag;
+
+ public TermSource(TermSource other) {
+ this.iter = other.iter;
+ this.term = other.term;
+ this.notFlag = other.notFlag;
+ this.seekColfams = other.seekColfams;
+ }
+
+ public TermSource(SortedKeyValueIterator<Key,Value> iter, Text term) {
+ this(iter, term, false);
+ }
+
+ public TermSource(SortedKeyValueIterator<Key,Value> iter, Text term, boolean notFlag) {
+ this.iter = iter;
+ this.term = term;
+ this.notFlag = notFlag;
+ // The desired column families for this source is the term itself
+ this.seekColfams = Collections.<ByteSequence> singletonList(new ArrayByteSequence(term.getBytes(), 0, term.getLength()));
+ }
+
+ public String getTermString() {
+ return (this.term == null) ? "Iterator" : this.term.toString();
+ }
+ }
+
+ protected TermSource[] sources;
+ int sourcesCount = 0;
+
+ Range overallRange;
+
+ // query-time settings
+ protected Text currentPartition = null;
+ protected Text currentDocID = new Text(emptyByteArray);
+ static final byte[] emptyByteArray = new byte[0];
+
+ protected Key topKey = null;
+ protected Value value = new Value(emptyByteArray);
+
+ public IntersectingIterator() {}
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ return new IntersectingIterator(this, env);
+ }
+
+ private IntersectingIterator(IntersectingIterator other, IteratorEnvironment env) {
+ if (other.sources != null) {
+ sourcesCount = other.sourcesCount;
+ sources = new TermSource[sourcesCount];
+ for (int i = 0; i < sourcesCount; i++) {
+ sources[i] = new TermSource(other.sources[i].iter.deepCopy(env), other.sources[i].term);
+ }
+ }
+ }
+
+ @Override
+ public Key getTopKey() {
+ return topKey;
+ }
+
+ @Override
+ public Value getTopValue() {
+ // we don't really care about values
+ return value;
+ }
+
+ @Override
+ public boolean hasTop() {
+ return currentPartition != null;
+ }
+
+ // precondition: currentRow is not null
+ private boolean seekOneSource(int sourceID) throws IOException {
+ // find the next key in the appropriate column family that is at or beyond the cursor (currentRow, currentCQ)
+ // advance the cursor if this source goes beyond it
+ // return whether we advanced the cursor
+
+ // within this loop progress must be made in one of the following forms:
+ // - currentRow or currentCQ must be increased
+ // - the given source must advance its iterator
+ // this loop will end when any of the following criteria are met
+ // - the iterator for the given source is pointing to the key (currentRow, columnFamilies[sourceID], currentCQ)
+ // - the given source is out of data and currentRow is set to null
+ // - the given source has advanced beyond the endRow and currentRow is set to null
+ boolean advancedCursor = false;
+
+ if (sources[sourceID].notFlag) {
+ while (true) {
+ if (sources[sourceID].iter.hasTop() == false) {
+ // an empty column that you are negating is a valid condition
+ break;
+ }
+ // check if we're past the end key
+ int endCompare = -1;
+ // we should compare the row to the end of the range
+ if (overallRange.getEndKey() != null) {
+ endCompare = overallRange.getEndKey().getRow().compareTo(sources[sourceID].iter.getTopKey().getRow());
+ if ((!overallRange.isEndKeyInclusive() && endCompare <= 0) || endCompare < 0) {
+ // an empty column that you are negating is a valid condition
+ break;
+ }
+ }
+ int partitionCompare = currentPartition.compareTo(getPartition(sources[sourceID].iter.getTopKey()));
+ // check if this source is already at or beyond currentRow
+ // if not, then seek to at least the current row
+
+ if (partitionCompare > 0) {
+ // seek to at least the currentRow
+ Key seekKey = buildKey(currentPartition, sources[sourceID].term);
+ sources[sourceID].iter.seek(new Range(seekKey, true, null, false), sources[sourceID].seekColfams, true);
+ continue;
+ }
+ // check if this source has gone beyond currentRow
+ // if so, this is a valid condition for negation
+ if (partitionCompare < 0) {
+ break;
+ }
+ // we have verified that the current source is positioned in currentRow
+ // now we must make sure we're in the right columnFamily in the current row
+ // Note: Iterators are auto-magically set to the correct columnFamily
+ if (sources[sourceID].term != null) {
+ int termCompare = sources[sourceID].term.compareTo(getTerm(sources[sourceID].iter.getTopKey()));
+ // check if this source is already on the right columnFamily
+ // if not, then seek forwards to the right columnFamily
+ if (termCompare > 0) {
+ Key seekKey = buildKey(currentPartition, sources[sourceID].term, currentDocID);
+ sources[sourceID].iter.seek(new Range(seekKey, true, null, false), sources[sourceID].seekColfams, true);
+ continue;
+ }
+ // check if this source is beyond the right columnFamily
+ // if so, then this is a valid condition for negating
+ if (termCompare < 0) {
+ break;
+ }
+ }
+
+ // we have verified that we are in currentRow and the correct column family
+ // make sure we are at or beyond columnQualifier
+ Text docID = getDocID(sources[sourceID].iter.getTopKey());
+ int docIDCompare = currentDocID.compareTo(docID);
+ // If we are past the target, this is a valid result
+ if (docIDCompare < 0) {
+ break;
+ }
+ // if this source is not yet at the currentCQ then advance in this source
+ if (docIDCompare > 0) {
+ // seek forwards
+ Key seekKey = buildKey(currentPartition, sources[sourceID].term, currentDocID);
+ sources[sourceID].iter.seek(new Range(seekKey, true, null, false), sources[sourceID].seekColfams, true);
+ continue;
+ }
+ // if we are equal to the target, this is an invalid result.
+ // Force the entire process to go to the next row.
+ // We are advancing column 0 because we forced that column to not contain a !
+ // when we did the init()
+ if (docIDCompare == 0) {
+ sources[0].iter.next();
+ advancedCursor = true;
+ break;
+ }
+ }
+ } else {
+ while (true) {
+ if (sources[sourceID].iter.hasTop() == false) {
+ currentPartition = null;
+ // setting currentRow to null counts as advancing the cursor
+ return true;
+ }
+ // check if we're past the end key
+ int endCompare = -1;
+ // we should compare the row to the end of the range
+
+ if (overallRange.getEndKey() != null) {
+ endCompare = overallRange.getEndKey().getRow().compareTo(sources[sourceID].iter.getTopKey().getRow());
+ if ((!overallRange.isEndKeyInclusive() && endCompare <= 0) || endCompare < 0) {
+ currentPartition = null;
+ // setting currentRow to null counts as advancing the cursor
+ return true;
+ }
+ }
+ int partitionCompare = currentPartition.compareTo(getPartition(sources[sourceID].iter.getTopKey()));
+ // check if this source is already at or beyond currentRow
+ // if not, then seek to at least the current row
+ if (partitionCompare > 0) {
+ // seek to at least the currentRow
+ Key seekKey = buildKey(currentPartition, sources[sourceID].term);
+ sources[sourceID].iter.seek(new Range(seekKey, true, null, false), sources[sourceID].seekColfams, true);
+ continue;
+ }
+ // check if this source has gone beyond currentRow
+ // if so, advance currentRow
+ if (partitionCompare < 0) {
+ currentPartition.set(getPartition(sources[sourceID].iter.getTopKey()));
+ currentDocID.set(emptyByteArray);
+ advancedCursor = true;
+ continue;
+ }
+ // we have verified that the current source is positioned in currentRow
+ // now we must make sure we're in the right columnFamily in the current row
+ // Note: Iterators are auto-magically set to the correct columnFamily
+
+ if (sources[sourceID].term != null) {
+ int termCompare = sources[sourceID].term.compareTo(getTerm(sources[sourceID].iter.getTopKey()));
+ // check if this source is already on the right columnFamily
+ // if not, then seek forwards to the right columnFamily
+ if (termCompare > 0) {
+ Key seekKey = buildKey(currentPartition, sources[sourceID].term, currentDocID);
+ sources[sourceID].iter.seek(new Range(seekKey, true, null, false), sources[sourceID].seekColfams, true);
+ continue;
+ }
+ // check if this source is beyond the right columnFamily
+ // if so, then seek to the next row
+ if (termCompare < 0) {
+ // we're out of entries in the current row, so seek to the next one
+ // byte[] currentRowBytes = currentRow.getBytes();
+ // byte[] nextRow = new byte[currentRowBytes.length + 1];
+ // System.arraycopy(currentRowBytes, 0, nextRow, 0, currentRowBytes.length);
+ // nextRow[currentRowBytes.length] = (byte)0;
+ // // we should reuse text objects here
+ // sources[sourceID].seek(new Key(new Text(nextRow),columnFamilies[sourceID]));
+ if (endCompare == 0) {
+ // we're done
+ currentPartition = null;
+ // setting currentRow to null counts as advancing the cursor
+ return true;
+ }
+ Key seekKey = buildFollowingPartitionKey(sources[sourceID].iter.getTopKey());
+ sources[sourceID].iter.seek(new Range(seekKey, true, null, false), sources[sourceID].seekColfams, true);
+ continue;
+ }
+ }
+ // we have verified that we are in currentRow and the correct column family
+ // make sure we are at or beyond columnQualifier
+ Text docID = getDocID(sources[sourceID].iter.getTopKey());
+ int docIDCompare = currentDocID.compareTo(docID);
+ // if this source has advanced beyond the current column qualifier then advance currentCQ and return true
+ if (docIDCompare < 0) {
+ currentDocID.set(docID);
+ advancedCursor = true;
+ break;
+ }
+ // if this source is not yet at the currentCQ then seek in this source
+ if (docIDCompare > 0) {
+ // seek forwards
+ Key seekKey = buildKey(currentPartition, sources[sourceID].term, currentDocID);
+ sources[sourceID].iter.seek(new Range(seekKey, true, null, false), sources[sourceID].seekColfams, true);
+ continue;
+ }
+ // this source is at the current row, in its column family, and at currentCQ
+ break;
+ }
+ }
+ return advancedCursor;
+ }
+
+ @Override
+ public void next() throws IOException {
+ if (currentPartition == null) {
+ return;
+ }
+ // precondition: the current row is set up and the sources all have the same column qualifier
+ // while we don't have a match, seek in the source with the smallest column qualifier
+ sources[0].iter.next();
+ advanceToIntersection();
+ }
+
+ protected void advanceToIntersection() throws IOException {
+ boolean cursorChanged = true;
+ while (cursorChanged) {
+ // seek all of the sources to at least the highest seen column qualifier in the current row
+ cursorChanged = false;
+ for (int i = 0; i < sourcesCount; i++) {
+ if (currentPartition == null) {
+ topKey = null;
+ return;
+ }
+ if (seekOneSource(i)) {
+ cursorChanged = true;
+ break;
+ }
+ }
+ }
+ topKey = buildKey(currentPartition, nullText, currentDocID);
+ }
+
+ public static String stringTopKey(SortedKeyValueIterator<Key,Value> iter) {
+ if (iter.hasTop())
+ return iter.getTopKey().toString();
+ return "";
+ }
+
+ private static final String columnFamiliesOptionName = "columnFamilies";
+ private static final String notFlagOptionName = "notFlag";
+
+ /**
- * @param columns
+ * @return encoded columns
+ */
+ protected static String encodeColumns(Text[] columns) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < columns.length; i++) {
+ sb.append(new String(Base64.encodeBase64(TextUtil.getBytes(columns[i])), Constants.UTF8));
+ sb.append('\n');
+ }
+ return sb.toString();
+ }
+
+ /**
- * @param flags
+ * @return encoded flags
+ */
+ protected static String encodeBooleans(boolean[] flags) {
+ byte[] bytes = new byte[flags.length];
+ for (int i = 0; i < flags.length; i++) {
+ if (flags[i])
+ bytes[i] = 1;
+ else
+ bytes[i] = 0;
+ }
+ return new String(Base64.encodeBase64(bytes), Constants.UTF8);
+ }
+
+ protected static Text[] decodeColumns(String columns) {
+ String[] columnStrings = columns.split("\n");
+ Text[] columnTexts = new Text[columnStrings.length];
+ for (int i = 0; i < columnStrings.length; i++) {
+ columnTexts[i] = new Text(Base64.decodeBase64(columnStrings[i].getBytes(Constants.UTF8)));
+ }
+ return columnTexts;
+ }
+
+ /**
- * @param flags
+ * @return decoded flags
+ */
+ protected static boolean[] decodeBooleans(String flags) {
+ // return null of there were no flags
+ if (flags == null)
+ return null;
+
+ byte[] bytes = Base64.decodeBase64(flags.getBytes(Constants.UTF8));
+ boolean[] bFlags = new boolean[bytes.length];
+ for (int i = 0; i < bytes.length; i++) {
+ if (bytes[i] == 1)
+ bFlags[i] = true;
+ else
+ bFlags[i] = false;
+ }
+ return bFlags;
+ }
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ Text[] terms = decodeColumns(options.get(columnFamiliesOptionName));
+ boolean[] notFlag = decodeBooleans(options.get(notFlagOptionName));
+
+ if (terms.length < 2) {
+ throw new IllegalArgumentException("IntersectionIterator requires two or more columns families");
+ }
+
+ // Scan the not flags.
+ // There must be at least one term that isn't negated
+ // And we are going to re-order such that the first term is not a ! term
+ if (notFlag == null) {
+ notFlag = new boolean[terms.length];
+ for (int i = 0; i < terms.length; i++)
+ notFlag[i] = false;
+ }
+ if (notFlag[0]) {
+ for (int i = 1; i < notFlag.length; i++) {
+ if (notFlag[i] == false) {
+ Text swapFamily = new Text(terms[0]);
+ terms[0].set(terms[i]);
+ terms[i].set(swapFamily);
+ notFlag[0] = false;
+ notFlag[i] = true;
+ break;
+ }
+ }
+ if (notFlag[0]) {
+ throw new IllegalArgumentException("IntersectionIterator requires at lest one column family without not");
+ }
+ }
+
+ sources = new TermSource[terms.length];
+ sources[0] = new TermSource(source, terms[0]);
+ for (int i = 1; i < terms.length; i++) {
+ sources[i] = new TermSource(source.deepCopy(env), terms[i], notFlag[i]);
+ }
+ sourcesCount = terms.length;
+ }
+
+ @Override
+ public void seek(Range range, Collection<ByteSequence> seekColumnFamilies, boolean inclusive) throws IOException {
+ overallRange = new Range(range);
+ currentPartition = new Text();
+ currentDocID.set(emptyByteArray);
+
+ // seek each of the sources to the right column family within the row given by key
+ for (int i = 0; i < sourcesCount; i++) {
+ Key sourceKey;
+ if (range.getStartKey() != null) {
+ if (range.getStartKey().getColumnQualifier() != null) {
+ sourceKey = buildKey(getPartition(range.getStartKey()), sources[i].term, range.getStartKey().getColumnQualifier());
+ } else {
+ sourceKey = buildKey(getPartition(range.getStartKey()), sources[i].term);
+ }
+ // Seek only to the term for this source as a column family
+ sources[i].iter.seek(new Range(sourceKey, true, null, false), sources[i].seekColfams, true);
+ } else {
+ // Seek only to the term for this source as a column family
+ sources[i].iter.seek(range, sources[i].seekColfams, true);
+ }
+ }
+ advanceToIntersection();
+ }
+
+ public void addSource(SortedKeyValueIterator<Key,Value> source, IteratorEnvironment env, Text term, boolean notFlag) {
+ // Check if we have space for the added Source
+ if (sources == null) {
+ sources = new TermSource[1];
+ } else {
+ // allocate space for node, and copy current tree.
+ // TODO: Should we change this to an ArrayList so that we can just add() ? - ACCUMULO-1309
+ TermSource[] localSources = new TermSource[sources.length + 1];
+ int currSource = 0;
+ for (TermSource myTerm : sources) {
+ // TODO: Do I need to call new here? or can I just re-use the term? - ACCUMULO-1309
+ localSources[currSource] = new TermSource(myTerm);
+ currSource++;
+ }
+ sources = localSources;
+ }
+ sources[sourcesCount] = new TermSource(source.deepCopy(env), term, notFlag);
+ sourcesCount++;
+ }
+
+ /**
+ * Encode the columns to be used when iterating.
- *
- * @param cfg
- * @param columns
+ */
+ public static void setColumnFamilies(IteratorSetting cfg, Text[] columns) {
+ if (columns.length < 2)
+ throw new IllegalArgumentException("Must supply at least two terms to intersect");
+ cfg.addOption(IntersectingIterator.columnFamiliesOptionName, IntersectingIterator.encodeColumns(columns));
+ }
+
+ /**
+ * Encode columns and NOT flags indicating which columns should be negated (docIDs will be excluded if matching negated columns, instead of included).
- *
- * @param cfg
- * @param columns
- * @param notFlags
+ */
+ public static void setColumnFamilies(IteratorSetting cfg, Text[] columns, boolean[] notFlags) {
+ if (columns.length < 2)
+ throw new IllegalArgumentException("Must supply at least two terms to intersect");
+ if (columns.length != notFlags.length)
+ throw new IllegalArgumentException("columns and notFlags arrays must be the same length");
+ setColumnFamilies(cfg, columns);
+ cfg.addOption(IntersectingIterator.notFlagOptionName, IntersectingIterator.encodeBooleans(notFlags));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/user/RowFilter.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/RowFilter.java
index a232796,0000000..2d2fa74
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/RowFilter.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/RowFilter.java
@@@ -1,165 -1,0 +1,164 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators.user;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.WrappingIterator;
+import org.apache.hadoop.io.Text;
+
+/**
+ * This iterator makes it easy to select rows that meet a given criteria. Its an alternative to the {@link WholeRowIterator}. There are a few things to consider
+ * when deciding which one to use.
+ *
+ * First the WholeRowIterator requires that the row fit in memory and that the entire row is read before a decision is made. This iterator has neither
+ * requirement, it allows seeking within a row to avoid reading the entire row to make a decision. So even if your rows fit into memory, this extending this
+ * iterator may be better choice because you can seek.
+ *
+ * Second the WholeRowIterator is currently the only way to achieve row isolation with the {@link BatchScanner}. With the normal {@link Scanner} row isolation
+ * can be enabled and this Iterator may be used.
+ *
+ * Third the row acceptance test will be executed every time this Iterator is seeked. If the row is large, then the row will fetched in batches of key/values.
+ * As each batch is fetched the test may be re-executed because the iterator stack is reseeked for each batch. The batch size may be increased to reduce the
+ * number of times the test is executed. With the normal Scanner, if isolation is enabled then it will read an entire row w/o seeking this iterator.
+ *
+ */
+public abstract class RowFilter extends WrappingIterator {
+
+ private RowIterator decisionIterator;
+ private Collection<ByteSequence> columnFamilies;
+ Text currentRow;
+ private boolean inclusive;
+ private Range range;
+ private boolean hasTop;
+
+ private static class RowIterator extends WrappingIterator {
+ private Range rowRange;
+ private boolean hasTop;
+
+ RowIterator(SortedKeyValueIterator<Key,Value> source) {
+ super.setSource(source);
+ }
+
+ void setRow(Range row) {
+ this.rowRange = row;
+ }
+
+ @Override
+ public boolean hasTop() {
+ return hasTop && super.hasTop();
+ }
+
+ @Override
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
+
+ range = rowRange.clip(range, true);
+ if (range == null) {
+ hasTop = false;
+ } else {
+ hasTop = true;
+ super.seek(range, columnFamilies, inclusive);
+ }
+ }
+ }
+
+ private void skipRows() throws IOException {
+ SortedKeyValueIterator<Key,Value> source = getSource();
+ while (source.hasTop()) {
+ Text row = source.getTopKey().getRow();
+
+ if (currentRow != null && currentRow.equals(row))
+ break;
+
+ Range rowRange = new Range(row);
+ decisionIterator.setRow(rowRange);
+ decisionIterator.seek(rowRange, columnFamilies, inclusive);
+
+ if (acceptRow(decisionIterator)) {
+ currentRow = row;
+ break;
+ } else {
+ currentRow = null;
+ int count = 0;
+ while (source.hasTop() && count < 10 && source.getTopKey().getRow().equals(row)) {
+ count++;
+ source.next();
+ }
+
+ if (source.hasTop() && source.getTopKey().getRow().equals(row)) {
+ Range nextRow = new Range(row, false, null, false);
+ nextRow = range.clip(nextRow, true);
+ if (nextRow == null)
+ hasTop = false;
+ else
+ source.seek(nextRow, columnFamilies, inclusive);
+ }
+ }
+ }
+ }
+
+ /**
+ * Implementation should return false to suppress a row.
+ *
+ *
+ * @param rowIterator
+ * - An iterator over the row. This iterator is confined to the row. Seeking past the end of the row will return no data. Seeking before the row will
+ * always set top to the first column in the current row. By default this iterator will only see the columns the parent was seeked with. To see more
+ * columns reseek this iterator with those columns.
+ * @return false if a row should be suppressed, otherwise true.
- * @throws IOException
+ */
+ public abstract boolean acceptRow(SortedKeyValueIterator<Key,Value> rowIterator) throws IOException;
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ super.init(source, options, env);
+ this.decisionIterator = new RowIterator(source.deepCopy(env));
+ }
+
+ @Override
+ public boolean hasTop() {
+ return hasTop && super.hasTop();
+ }
+
+ @Override
+ public void next() throws IOException {
+ super.next();
+ skipRows();
+ }
+
+ @Override
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
+ super.seek(range, columnFamilies, inclusive);
+ this.columnFamilies = columnFamilies;
+ this.inclusive = inclusive;
+ this.range = range;
+ currentRow = null;
+ hasTop = true;
+ skipRows();
+
+ }
+}
[28/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/data/Range.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Range.java
index 65873c3,0000000..122436b
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/data/Range.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Range.java
@@@ -1,906 -1,0 +1,899 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
+import java.io.DataInput;
+import java.io.DataOutput;
- import java.io.InvalidObjectException;
+import java.io.IOException;
++import java.io.InvalidObjectException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.accumulo.core.data.thrift.TRange;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+
+/**
+ * This class is used to specify a range of Accumulo Keys.
+ *
+ */
+
+public class Range implements WritableComparable<Range> {
+
+ private Key start;
+ private Key stop;
+ private boolean startKeyInclusive;
+ private boolean stopKeyInclusive;
+ private boolean infiniteStartKey;
+ private boolean infiniteStopKey;
+
+ /**
+ * Creates a range that goes from negative to positive infinity
+ */
+
+ public Range() {
+ this((Key) null, true, (Key) null, true);
+ }
+
+ /**
+ * Creates a range from startKey inclusive to endKey inclusive
+ *
+ * @param startKey
+ * set this to null when negative infinity is needed
+ * @param endKey
+ * set this to null when positive infinity is needed
+ */
+ public Range(Key startKey, Key endKey) {
+ this(startKey, true, endKey, true);
+ }
+
+ /**
+ * Creates a range that covers an entire row
+ *
+ * @param row
+ * set this to null to cover all rows
+ */
+ public Range(CharSequence row) {
+ this(row, true, row, true);
+ }
+
+ /**
+ * Creates a range that covers an entire row
+ *
+ * @param row
+ * set this to null to cover all rows
+ */
+ public Range(Text row) {
+ this(row, true, row, true);
+ }
+
+ /**
+ * Creates a range from startRow inclusive to endRow inclusive
+ *
+ * @param startRow
+ * set this to null when negative infinity is needed
+ * @param endRow
+ * set this to null when positive infinity is needed
+ */
+ public Range(Text startRow, Text endRow) {
+ this(startRow, true, endRow, true);
+ }
+
+ /**
+ * Creates a range from startRow inclusive to endRow inclusive
+ *
+ * @param startRow
+ * set this to null when negative infinity is needed
+ * @param endRow
+ * set this to null when positive infinity is needed
+ */
+ public Range(CharSequence startRow, CharSequence endRow) {
+ this(startRow, true, endRow, true);
+ }
+
+ /**
+ * Creates a range from startRow to endRow
+ *
+ * @param startRow
+ * set this to null when negative infinity is needed
+ * @param startRowInclusive
+ * determines if the start row is skipped
+ * @param endRow
+ * set this to null when positive infinity is needed
+ * @param endRowInclusive
+ * determines if the endRow is included
+ */
+
+ public Range(Text startRow, boolean startRowInclusive, Text endRow, boolean endRowInclusive) {
+ this((startRow == null ? null : (startRowInclusive ? new Key(startRow) : new Key(startRow).followingKey(PartialKey.ROW))), true, (endRow == null ? null
+ : (endRowInclusive ? new Key(endRow).followingKey(PartialKey.ROW) : new Key(endRow))), false);
+ }
+
+ /**
+ * Creates a range from startRow to endRow
+ *
+ * @param startRow
+ * set this to null when negative infinity is needed
+ * @param startRowInclusive
+ * determines if the start row is skipped
+ * @param endRow
+ * set this to null when positive infinity is needed
+ * @param endRowInclusive
+ * determines if the endRow is included
+ */
+
+ public Range(CharSequence startRow, boolean startRowInclusive, CharSequence endRow, boolean endRowInclusive) {
+ this(startRow == null ? null : new Text(startRow.toString()), startRowInclusive, endRow == null ? null : new Text(endRow.toString()), endRowInclusive);
+ }
+
+ /**
+ * Creates a range from startKey to endKey
+ *
+ * @param startKey
+ * set this to null when negative infinity is needed
+ * @param startKeyInclusive
+ * determines if the ranges includes the start key
+ * @param endKey
+ * set this to null when infinity is needed
+ * @param endKeyInclusive
+ * determines if the range includes the end key
+ */
+ public Range(Key startKey, boolean startKeyInclusive, Key endKey, boolean endKeyInclusive) {
+ this.start = startKey;
+ this.startKeyInclusive = startKeyInclusive;
+ this.infiniteStartKey = startKey == null;
+ this.stop = endKey;
+ this.stopKeyInclusive = endKeyInclusive;
+ this.infiniteStopKey = stop == null;
+
+ if (!infiniteStartKey && !infiniteStopKey && beforeStartKey(endKey)) {
+ throw new IllegalArgumentException("Start key must be less than end key in range (" + startKey + ", " + endKey + ")");
+ }
+ }
+
+ /**
+ * Copies a range
+ */
+ public Range(Range range) {
+ this(range.start, range.startKeyInclusive, range.infiniteStartKey, range.stop, range.stopKeyInclusive, range.infiniteStopKey);
+ }
+
+ /**
+ * Creates a range from start to stop.
+ *
+ * @param start
+ * set this to null when negative infinity is needed
+ * @param stop
+ * set this to null when infinity is needed
+ * @param startKeyInclusive
+ * determines if the ranges includes the start key
+ * @param stopKeyInclusive
+ * determines if the range includes the end key
+ * @param infiniteStartKey
+ * true if start key is negative infinity (null)
+ * @param infiniteStopKey
+ * true if stop key is positive infinity (null)
+ * @throws IllegalArgumentException if stop is before start, or infiniteStartKey is true but start is not null, or infiniteStopKey is true but stop is not
+ * null
+ */
+ public Range(Key start, Key stop, boolean startKeyInclusive, boolean stopKeyInclusive, boolean infiniteStartKey, boolean infiniteStopKey) {
+ this(start, startKeyInclusive, infiniteStartKey, stop, stopKeyInclusive, infiniteStopKey);
+ if (!infiniteStartKey && !infiniteStopKey && beforeStartKey(stop)) {
+ throw new IllegalArgumentException("Start key must be less than end key in range (" + start + ", " + stop + ")");
+ }
+ }
+
+ /**
+ * Creates a range from start to stop. Unlike the public six-argument method,
+ * this one does not assure that stop is after start, which helps performance
+ * in cases where that assurance is already in place.
+ *
+ * @param start
+ * set this to null when negative infinity is needed
+ * @param startKeyInclusive
+ * determines if the ranges includes the start key
+ * @param infiniteStartKey
+ * true if start key is negative infinity (null)
+ * @param stop
+ * set this to null when infinity is needed
+ * @param stopKeyInclusive
+ * determines if the range includes the end key
+ * @param infiniteStopKey
+ * true if stop key is positive infinity (null)
+ * @throws IllegalArgumentException if infiniteStartKey is true but start is not null, or infiniteStopKey is true but stop is not null
+ */
+ protected Range(Key start, boolean startKeyInclusive, boolean infiniteStartKey, Key stop, boolean stopKeyInclusive, boolean infiniteStopKey) {
+ if (infiniteStartKey && start != null)
+ throw new IllegalArgumentException();
+
+ if (infiniteStopKey && stop != null)
+ throw new IllegalArgumentException();
+
+ this.start = start;
+ this.stop = stop;
+ this.startKeyInclusive = startKeyInclusive;
+ this.stopKeyInclusive = stopKeyInclusive;
+ this.infiniteStartKey = infiniteStartKey;
+ this.infiniteStopKey = infiniteStopKey;
+ }
+
+ public Range(TRange trange) {
+ this(trange.start == null ? null : new Key(trange.start), trange.startKeyInclusive, trange.infiniteStartKey,
+ trange.stop == null ? null : new Key(trange.stop), trange.stopKeyInclusive, trange.infiniteStopKey);
+ if (!infiniteStartKey && !infiniteStopKey && beforeStartKey(stop)) {
+ throw new IllegalArgumentException("Start key must be less than end key in range (" + start + ", " + stop + ")");
+ }
+ }
+
+ /**
+ * @return the first key in the range, null if the key is infinite
+ */
+ public Key getStartKey() {
+ if (infiniteStartKey) {
+ return null;
+ }
+ return start;
+ }
+
+ /**
+ *
- * @param key
+ * @return true if the given key is before the range, otherwise false
+ */
+ public boolean beforeStartKey(Key key) {
+ if (infiniteStartKey) {
+ return false;
+ }
+
+ if (startKeyInclusive)
+ return key.compareTo(start) < 0;
+ return key.compareTo(start) <= 0;
+ }
+
+ /**
+ * @return the last key in the range, null if the end key is infinite
+ */
+
+ public Key getEndKey() {
+ if (infiniteStopKey) {
+ return null;
+ }
+ return stop;
+ }
+
+ /**
- * @param key
+ * @return true if the given key is after the range, otherwise false
+ */
+
+ public boolean afterEndKey(Key key) {
+ if (infiniteStopKey)
+ return false;
+
+ if (stopKeyInclusive)
+ return stop.compareTo(key) < 0;
+ return stop.compareTo(key) <= 0;
+ }
+
+ @Override
+ public int hashCode() {
+ int startHash = infiniteStartKey ? 0 : start.hashCode() + (startKeyInclusive ? 1 : 0);
+ int stopHash = infiniteStopKey ? 0 : stop.hashCode() + (stopKeyInclusive ? 1 : 0);
+
+ return startHash + stopHash;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof Range)
+ return equals((Range) o);
+ return false;
+ }
+
+ public boolean equals(Range otherRange) {
+
+ return compareTo(otherRange) == 0;
+ }
+
+ /**
+ * Compares this range to another range. Compares in the order start key, inclusiveness of start key, end key, inclusiveness of end key. Infinite keys sort
+ * first, and non-infinite keys are compared with {@link Key#compareTo(Key)}. Inclusive sorts before non-inclusive.
+ */
++ @Override
+ public int compareTo(Range o) {
+ int comp;
+
+ if (infiniteStartKey)
+ if (o.infiniteStartKey)
+ comp = 0;
+ else
+ comp = -1;
+ else if (o.infiniteStartKey)
+ comp = 1;
+ else {
+ comp = start.compareTo(o.start);
+ if (comp == 0)
+ if (startKeyInclusive && !o.startKeyInclusive)
+ comp = -1;
+ else if (!startKeyInclusive && o.startKeyInclusive)
+ comp = 1;
+
+ }
+
+ if (comp == 0)
+ if (infiniteStopKey)
+ if (o.infiniteStopKey)
+ comp = 0;
+ else
+ comp = 1;
+ else if (o.infiniteStopKey)
+ comp = -1;
+ else {
+ comp = stop.compareTo(o.stop);
+ if (comp == 0)
+ if (stopKeyInclusive && !o.stopKeyInclusive)
+ comp = 1;
+ else if (!stopKeyInclusive && o.stopKeyInclusive)
+ comp = -1;
+ }
+
+ return comp;
+ }
+
+ /**
+ *
- * @param key
+ * @return true if the given key falls within the range
+ */
+ public boolean contains(Key key) {
+ return !beforeStartKey(key) && !afterEndKey(key);
+ }
+
+ /**
+ * Takes a collection on range and merges overlapping and adjacent ranges. For example given the following input
+ *
+ * <pre>
+ * [a,c], (c, d], (g,m), (j,t]
+ * </pre>
+ *
+ * the following ranges would be returned
+ *
+ * <pre>
+ * [a,d], (g,t]
+ * </pre>
+ *
- * @param ranges
+ * @return list of merged ranges
+ */
+
+ public static List<Range> mergeOverlapping(Collection<Range> ranges) {
+ if (ranges.size() == 0)
+ return Collections.emptyList();
+
+ List<Range> ral = new ArrayList<Range>(ranges);
+ Collections.sort(ral);
+
+ ArrayList<Range> ret = new ArrayList<Range>(ranges.size());
+
+ Range currentRange = ral.get(0);
+ boolean currentStartKeyInclusive = ral.get(0).startKeyInclusive;
+
+ for (int i = 1; i < ral.size(); i++) {
+ // because of inclusive switch, equal keys may not be seen
+
+ if (currentRange.infiniteStopKey) {
+ // this range has the minimal start key and
+ // an infinite end key so it will contain all
+ // other ranges
+ break;
+ }
+
+ Range range = ral.get(i);
+
+ boolean startKeysEqual;
+ if (range.infiniteStartKey) {
+ // previous start key must be infinite because it is sorted
+ assert (currentRange.infiniteStartKey);
+ startKeysEqual = true;
+ } else if (currentRange.infiniteStartKey) {
+ startKeysEqual = false;
+ } else if (currentRange.start.equals(range.start)) {
+ startKeysEqual = true;
+ } else {
+ startKeysEqual = false;
+ }
+
+ if (startKeysEqual || currentRange.contains(range.start)
+ || (!currentRange.stopKeyInclusive && range.startKeyInclusive && range.start.equals(currentRange.stop))) {
+ int cmp;
+
+ if (range.infiniteStopKey || (cmp = range.stop.compareTo(currentRange.stop)) > 0 || (cmp == 0 && range.stopKeyInclusive)) {
+ currentRange = new Range(currentRange.getStartKey(), currentStartKeyInclusive, range.getEndKey(), range.stopKeyInclusive);
+ }/* else currentRange contains ral.get(i) */
+ } else {
+ ret.add(currentRange);
+ currentRange = range;
+ currentStartKeyInclusive = range.startKeyInclusive;
+ }
+ }
+
+ ret.add(currentRange);
+
+ return ret;
+ }
+
+ /**
+ * Creates a range which represents the intersection of this range and the passed in range. The following example will print true.
+ *
+ * <pre>
+ * Range range1 = new Range("a", "f");
+ * Range range2 = new Range("c", "n");
+ * Range range3 = range1.clip(range2);
+ * System.out.println(range3.equals(new Range("c", "f")));
+ * </pre>
+ *
- * @param range
+ * @return the intersection
+ * @throws IllegalArgumentException
+ * if ranges does not overlap
+ */
+
+ public Range clip(Range range) {
+ return clip(range, false);
+ }
+
+ /**
+ * Same as other clip function except if gives the option to return null of the ranges do not overlap instead of throwing an exception.
+ *
+ * @see Range#clip(Range)
- * @param range
+ * @param returnNullIfDisjoint
+ * If the ranges do not overlap and true is passed, then null is returned otherwise an exception is thrown.
+ * @return the intersection
+ */
+
+ public Range clip(Range range, boolean returnNullIfDisjoint) {
+
+ Key sk = range.getStartKey();
+ boolean ski = range.isStartKeyInclusive();
+
+ Key ek = range.getEndKey();
+ boolean eki = range.isEndKeyInclusive();
+
+ if (range.getStartKey() == null) {
+ if (getStartKey() != null) {
+ sk = getStartKey();
+ ski = isStartKeyInclusive();
+ }
+ } else if (afterEndKey(range.getStartKey())
+ || (getEndKey() != null && range.getStartKey().equals(getEndKey()) && !(range.isStartKeyInclusive() && isEndKeyInclusive()))) {
+ if (returnNullIfDisjoint)
+ return null;
+ throw new IllegalArgumentException("Range " + range + " does not overlap " + this);
+ } else if (beforeStartKey(range.getStartKey())) {
+ sk = getStartKey();
+ ski = isStartKeyInclusive();
+ }
+
+ if (range.getEndKey() == null) {
+ if (getEndKey() != null) {
+ ek = getEndKey();
+ eki = isEndKeyInclusive();
+ }
+ } else if (beforeStartKey(range.getEndKey())
+ || (getStartKey() != null && range.getEndKey().equals(getStartKey()) && !(range.isEndKeyInclusive() && isStartKeyInclusive()))) {
+ if (returnNullIfDisjoint)
+ return null;
+ throw new IllegalArgumentException("Range " + range + " does not overlap " + this);
+ } else if (afterEndKey(range.getEndKey())) {
+ ek = getEndKey();
+ eki = isEndKeyInclusive();
+ }
+
+ return new Range(sk, ski, ek, eki);
+ }
+
+ /**
+ * Creates a new range that is bounded by the columns passed in. The stary key in the returned range will have a column >= to the minimum column. The end key
+ * in the returned range will have a column <= the max column.
+ *
- *
- * @param min
- * @param max
+ * @return a column bounded range
+ * @throws IllegalArgumentException
+ * if min > max
+ */
+
+ public Range bound(Column min, Column max) {
+
+ if (min.compareTo(max) > 0) {
+ throw new IllegalArgumentException("min column > max column " + min + " " + max);
+ }
+
+ Key sk = getStartKey();
+ boolean ski = isStartKeyInclusive();
+
+ if (sk != null) {
+
+ ByteSequence cf = sk.getColumnFamilyData();
+ ByteSequence cq = sk.getColumnQualifierData();
+
+ ByteSequence mincf = new ArrayByteSequence(min.columnFamily);
+ ByteSequence mincq;
+
+ if (min.columnQualifier != null)
+ mincq = new ArrayByteSequence(min.columnQualifier);
+ else
+ mincq = new ArrayByteSequence(new byte[0]);
+
+ int cmp = cf.compareTo(mincf);
+
+ if (cmp < 0 || (cmp == 0 && cq.compareTo(mincq) < 0)) {
+ ski = true;
+ sk = new Key(sk.getRowData().toArray(), mincf.toArray(), mincq.toArray(), new byte[0], Long.MAX_VALUE, true);
+ }
+ }
+
+ Key ek = getEndKey();
+ boolean eki = isEndKeyInclusive();
+
+ if (ek != null) {
+ ByteSequence row = ek.getRowData();
+ ByteSequence cf = ek.getColumnFamilyData();
+ ByteSequence cq = ek.getColumnQualifierData();
+ ByteSequence cv = ek.getColumnVisibilityData();
+
+ ByteSequence maxcf = new ArrayByteSequence(max.columnFamily);
+ ByteSequence maxcq = null;
+ if (max.columnQualifier != null)
+ maxcq = new ArrayByteSequence(max.columnQualifier);
+
+ boolean set = false;
+
+ int comp = cf.compareTo(maxcf);
+
+ if (comp > 0) {
+ set = true;
+ } else if (comp == 0 && maxcq != null && cq.compareTo(maxcq) > 0) {
+ set = true;
+ } else if (!eki && row.length() > 0 && row.byteAt(row.length() - 1) == 0 && cf.length() == 0 && cq.length() == 0 && cv.length() == 0
+ && ek.getTimestamp() == Long.MAX_VALUE) {
+ row = row.subSequence(0, row.length() - 1);
+ set = true;
+ }
+
+ if (set) {
+ eki = false;
+ if (maxcq == null)
+ ek = new Key(row.toArray(), maxcf.toArray(), new byte[0], new byte[0], 0, false).followingKey(PartialKey.ROW_COLFAM);
+ else
+ ek = new Key(row.toArray(), maxcf.toArray(), maxcq.toArray(), new byte[0], 0, false).followingKey(PartialKey.ROW_COLFAM_COLQUAL);
+ }
+ }
+
+ return new Range(sk, ski, ek, eki);
+ }
+
++ @Override
+ public String toString() {
+ return ((startKeyInclusive && start != null) ? "[" : "(") + (start == null ? "-inf" : start) + "," + (stop == null ? "+inf" : stop)
+ + ((stopKeyInclusive && stop != null) ? "]" : ")");
+ }
+
++ @Override
+ public void readFields(DataInput in) throws IOException {
+ infiniteStartKey = in.readBoolean();
+ infiniteStopKey = in.readBoolean();
+ if (!infiniteStartKey) {
+ start = new Key();
+ start.readFields(in);
+ } else {
+ start = null;
+ }
+
+ if (!infiniteStopKey) {
+ stop = new Key();
+ stop.readFields(in);
+ } else {
+ stop = null;
+ }
+
+ startKeyInclusive = in.readBoolean();
+ stopKeyInclusive = in.readBoolean();
+
+ if (!infiniteStartKey && !infiniteStopKey && beforeStartKey(stop)) {
+ throw new InvalidObjectException("Start key must be less than end key in range (" + start + ", " + stop + ")");
+ }
+ }
+
++ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeBoolean(infiniteStartKey);
+ out.writeBoolean(infiniteStopKey);
+ if (!infiniteStartKey)
+ start.write(out);
+ if (!infiniteStopKey)
+ stop.write(out);
+ out.writeBoolean(startKeyInclusive);
+ out.writeBoolean(stopKeyInclusive);
+ }
+
+ public boolean isStartKeyInclusive() {
+ return startKeyInclusive;
+ }
+
+ public boolean isEndKeyInclusive() {
+ return stopKeyInclusive;
+ }
+
+ public TRange toThrift() {
+ return new TRange(start == null ? null : start.toThrift(), stop == null ? null : stop.toThrift(), startKeyInclusive, stopKeyInclusive, infiniteStartKey,
+ infiniteStopKey);
+ }
+
+ public boolean isInfiniteStartKey() {
+ return infiniteStartKey;
+ }
+
+ public boolean isInfiniteStopKey() {
+ return infiniteStopKey;
+ }
+
+ /**
+ * Creates a range that covers an exact row Returns the same Range as new Range(row)
+ *
+ * @param row
+ * all keys in the range will have this row
+ */
+ public static Range exact(Text row) {
+ return new Range(row);
+ }
+
+ /**
+ * Creates a range that covers an exact row and column family
+ *
+ * @param row
+ * all keys in the range will have this row
+ *
+ * @param cf
+ * all keys in the range will have this column family
+ */
+ public static Range exact(Text row, Text cf) {
+ Key startKey = new Key(row, cf);
+ return new Range(startKey, true, startKey.followingKey(PartialKey.ROW_COLFAM), false);
+ }
+
+ /**
+ * Creates a range that covers an exact row, column family, and column qualifier
+ *
+ * @param row
+ * all keys in the range will have this row
+ *
+ * @param cf
+ * all keys in the range will have this column family
+ *
+ * @param cq
+ * all keys in the range will have this column qualifier
+ */
+ public static Range exact(Text row, Text cf, Text cq) {
+ Key startKey = new Key(row, cf, cq);
+ return new Range(startKey, true, startKey.followingKey(PartialKey.ROW_COLFAM_COLQUAL), false);
+ }
+
+ /**
+ * Creates a range that covers an exact row, column family, column qualifier, and visibility
+ *
+ * @param row
+ * all keys in the range will have this row
+ *
+ * @param cf
+ * all keys in the range will have this column family
+ *
+ * @param cq
+ * all keys in the range will have this column qualifier
+ *
+ * @param cv
+ * all keys in the range will have this column visibility
+ */
+ public static Range exact(Text row, Text cf, Text cq, Text cv) {
+ Key startKey = new Key(row, cf, cq, cv);
+ return new Range(startKey, true, startKey.followingKey(PartialKey.ROW_COLFAM_COLQUAL_COLVIS), false);
+ }
+
+ /**
+ * Creates a range that covers an exact row, column family, column qualifier, visibility, and timestamp
+ *
+ * @param row
+ * all keys in the range will have this row
+ *
+ * @param cf
+ * all keys in the range will have this column family
+ *
+ * @param cq
+ * all keys in the range will have this column qualifier
+ *
+ * @param cv
+ * all keys in the range will have this column visibility
+ *
+ * @param ts
+ * all keys in the range will have this timestamp
+ */
+ public static Range exact(Text row, Text cf, Text cq, Text cv, long ts) {
+ Key startKey = new Key(row, cf, cq, cv, ts);
+ return new Range(startKey, true, startKey.followingKey(PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME), false);
+ }
+
+ /**
+ * Returns a Text that sorts just after all Texts beginning with a prefix
- *
- * @param prefix
+ */
+ public static Text followingPrefix(Text prefix) {
+ byte[] prefixBytes = prefix.getBytes();
+
+ // find the last byte in the array that is not 0xff
+ int changeIndex = prefix.getLength() - 1;
+ while (changeIndex >= 0 && prefixBytes[changeIndex] == (byte) 0xff)
+ changeIndex--;
+ if (changeIndex < 0)
+ return null;
+
+ // copy prefix bytes into new array
+ byte[] newBytes = new byte[changeIndex + 1];
+ System.arraycopy(prefixBytes, 0, newBytes, 0, changeIndex + 1);
+
+ // increment the selected byte
+ newBytes[changeIndex]++;
+ return new Text(newBytes);
+ }
+
+ /**
+ * Returns a Range that covers all rows beginning with a prefix
+ *
+ * @param rowPrefix
+ * all keys in the range will have rows that begin with this prefix
+ */
+ public static Range prefix(Text rowPrefix) {
+ Text fp = followingPrefix(rowPrefix);
+ return new Range(new Key(rowPrefix), true, fp == null ? null : new Key(fp), false);
+ }
+
+ /**
+ * Returns a Range that covers all column families beginning with a prefix within a given row
+ *
+ * @param row
+ * all keys in the range will have this row
+ *
+ * @param cfPrefix
+ * all keys in the range will have column families that begin with this prefix
+ */
+ public static Range prefix(Text row, Text cfPrefix) {
+ Text fp = followingPrefix(cfPrefix);
+ return new Range(new Key(row, cfPrefix), true, fp == null ? new Key(row).followingKey(PartialKey.ROW) : new Key(row, fp), false);
+ }
+
+ /**
+ * Returns a Range that covers all column qualifiers beginning with a prefix within a given row and column family
+ *
+ * @param row
+ * all keys in the range will have this row
+ *
+ * @param cf
+ * all keys in the range will have this column family
+ *
+ * @param cqPrefix
+ * all keys in the range will have column qualifiers that begin with this prefix
+ */
+ public static Range prefix(Text row, Text cf, Text cqPrefix) {
+ Text fp = followingPrefix(cqPrefix);
+ return new Range(new Key(row, cf, cqPrefix), true, fp == null ? new Key(row, cf).followingKey(PartialKey.ROW_COLFAM) : new Key(row, cf, fp), false);
+ }
+
+ /**
+ * Returns a Range that covers all column visibilities beginning with a prefix within a given row, column family, and column qualifier
+ *
+ * @param row
+ * all keys in the range will have this row
+ *
+ * @param cf
+ * all keys in the range will have this column family
+ *
+ * @param cq
+ * all keys in the range will have this column qualifier
+ *
+ * @param cvPrefix
+ * all keys in the range will have column visibilities that begin with this prefix
+ */
+ public static Range prefix(Text row, Text cf, Text cq, Text cvPrefix) {
+ Text fp = followingPrefix(cvPrefix);
+ return new Range(new Key(row, cf, cq, cvPrefix), true, fp == null ? new Key(row, cf, cq).followingKey(PartialKey.ROW_COLFAM_COLQUAL) : new Key(row, cf, cq,
+ fp), false);
+ }
+
+ /**
+ * Creates a range that covers an exact row
+ *
+ * @see Range#exact(Text)
+ */
+ public static Range exact(CharSequence row) {
+ return Range.exact(new Text(row.toString()));
+ }
+
+ /**
+ * Creates a range that covers an exact row and column family
+ *
+ * @see Range#exact(Text, Text)
+ */
+ public static Range exact(CharSequence row, CharSequence cf) {
+ return Range.exact(new Text(row.toString()), new Text(cf.toString()));
+ }
+
+ /**
+ * Creates a range that covers an exact row, column family, and column qualifier
+ *
+ * @see Range#exact(Text, Text, Text)
+ */
+ public static Range exact(CharSequence row, CharSequence cf, CharSequence cq) {
+ return Range.exact(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()));
+ }
+
+ /**
+ * Creates a range that covers an exact row, column family, column qualifier, and visibility
+ *
+ * @see Range#exact(Text, Text, Text, Text)
+ */
+ public static Range exact(CharSequence row, CharSequence cf, CharSequence cq, CharSequence cv) {
+ return Range.exact(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()), new Text(cv.toString()));
+ }
+
+ /**
+ * Creates a range that covers an exact row, column family, column qualifier, visibility, and timestamp
+ *
+ * @see Range#exact(Text, Text, Text, Text, long)
+ */
+ public static Range exact(CharSequence row, CharSequence cf, CharSequence cq, CharSequence cv, long ts) {
+ return Range.exact(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()), new Text(cv.toString()), ts);
+ }
+
+ /**
+ * Returns a Range that covers all rows beginning with a prefix
+ *
+ * @see Range#prefix(Text)
+ */
+ public static Range prefix(CharSequence rowPrefix) {
+ return Range.prefix(new Text(rowPrefix.toString()));
+ }
+
+ /**
+ * Returns a Range that covers all column families beginning with a prefix within a given row
+ *
+ * @see Range#prefix(Text, Text)
+ */
+ public static Range prefix(CharSequence row, CharSequence cfPrefix) {
+ return Range.prefix(new Text(row.toString()), new Text(cfPrefix.toString()));
+ }
+
+ /**
+ * Returns a Range that covers all column qualifiers beginning with a prefix within a given row and column family
+ *
+ * @see Range#prefix(Text, Text, Text)
+ */
+ public static Range prefix(CharSequence row, CharSequence cf, CharSequence cqPrefix) {
+ return Range.prefix(new Text(row.toString()), new Text(cf.toString()), new Text(cqPrefix.toString()));
+ }
+
+ /**
+ * Returns a Range that covers all column visibilities beginning with a prefix within a given row, column family, and column qualifier
+ *
+ * @see Range#prefix(Text, Text, Text, Text)
+ */
+ public static Range prefix(CharSequence row, CharSequence cf, CharSequence cq, CharSequence cvPrefix) {
+ return Range.prefix(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()), new Text(cvPrefix.toString()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java
index 2b3cdf5,0000000..0ac5308
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java
@@@ -1,181 -1,0 +1,176 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.file.rfile;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.blockfile.ABlockReader;
+import org.apache.accumulo.core.file.rfile.MultiLevelIndex.IndexEntry;
+
+/**
+ *
+ */
+public class BlockIndex {
+
+ public static BlockIndex getIndex(ABlockReader cacheBlock, IndexEntry indexEntry) throws IOException {
+
+ BlockIndex blockIndex = cacheBlock.getIndex(BlockIndex.class);
+
+ int accessCount = blockIndex.accessCount.incrementAndGet();
+
+ // 1 is a power of two, but do not care about it
+ if (accessCount >= 2 && isPowerOfTwo(accessCount)) {
+ blockIndex.buildIndex(accessCount, cacheBlock, indexEntry);
+ }
+
+ if (blockIndex.blockIndex != null)
+ return blockIndex;
+
+ return null;
+ }
+
+ private static boolean isPowerOfTwo(int x) {
+ return ((x > 0) && (x & (x - 1)) == 0);
+ }
+
+ private AtomicInteger accessCount = new AtomicInteger(0);
+ private volatile BlockIndexEntry[] blockIndex = null;
+
+ public static class BlockIndexEntry implements Comparable<BlockIndexEntry> {
+
+ private Key prevKey;
+ private int entriesLeft;
+ private int pos;
+
+ public BlockIndexEntry(int pos, int entriesLeft, Key prevKey) {
+ this.pos = pos;
+ this.entriesLeft = entriesLeft;
+ this.prevKey = prevKey;
+ }
+
- /**
- * @param key
- */
+ public BlockIndexEntry(Key key) {
+ this.prevKey = key;
+ }
-
-
+
+ public int getEntriesLeft() {
+ return entriesLeft;
+ }
+
+ @Override
+ public int compareTo(BlockIndexEntry o) {
+ return prevKey.compareTo(o.prevKey);
+ }
+
+ @Override
+ public String toString() {
+ return prevKey + " " + entriesLeft + " " + pos;
+ }
+
+ public Key getPrevKey() {
+ return prevKey;
+ }
+ }
+
+ public BlockIndexEntry seekBlock(Key startKey, ABlockReader cacheBlock) {
+
+ // get a local ref to the index, another thread could change it
+ BlockIndexEntry[] blockIndex = this.blockIndex;
+
+ int pos = Arrays.binarySearch(blockIndex, new BlockIndexEntry(startKey));
+
+ int index;
+
+ if (pos < 0) {
+ if (pos == -1)
+ return null; // less than the first key in index, did not index the first key in block so just return null... code calling this will scan from beginning
+ // of block
+ index = (pos * -1) - 2;
+ } else {
+ // found exact key in index
+ index = pos;
+ while (index > 0) {
+ if (blockIndex[index].getPrevKey().equals(startKey))
+ index--;
+ else
+ break;
+ }
+ }
+
+ // handle case where multiple keys in block are exactly the same, want to find the earliest key in the index
+ while (index - 1 > 0) {
+ if (blockIndex[index].getPrevKey().equals(blockIndex[index - 1].getPrevKey()))
+ index--;
+ else
+ break;
+
+ }
+
+ if (index == 0 && blockIndex[index].getPrevKey().equals(startKey))
+ return null;
+
+ BlockIndexEntry bie = blockIndex[index];
+ cacheBlock.seek(bie.pos);
+ return bie;
+ }
+
+ private synchronized void buildIndex(int indexEntries, ABlockReader cacheBlock, IndexEntry indexEntry) throws IOException {
+ cacheBlock.seek(0);
+
+ RelativeKey rk = new RelativeKey();
+ Value val = new Value();
+
+ int interval = indexEntry.getNumEntries() / indexEntries;
+
+ if (interval <= 32)
+ return;
+
+ // multiple threads could try to create the index with different sizes, do not replace a large index with a smaller one
+ if (this.blockIndex != null && this.blockIndex.length > indexEntries - 1)
+ return;
+
+ int count = 0;
+
+ ArrayList<BlockIndexEntry> index = new ArrayList<BlockIndexEntry>(indexEntries - 1);
+
+ while (count < (indexEntry.getNumEntries() - interval + 1)) {
+
+ Key myPrevKey = rk.getKey();
+ int pos = cacheBlock.getPosition();
+ rk.readFields(cacheBlock);
+ val.readFields(cacheBlock);
+
+ if (count > 0 && count % interval == 0) {
+ index.add(new BlockIndexEntry(pos, indexEntry.getNumEntries() - count, myPrevKey));
+ }
+
+ count++;
+ }
+
+ this.blockIndex = index.toArray(new BlockIndexEntry[index.size()]);
+
+ cacheBlock.seek(0);
+ }
+
+ BlockIndexEntry[] getIndexEntries() {
+ return blockIndex;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index 7277c65,0000000..7d15851
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@@ -1,971 -1,0 +1,965 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.accumulo.core.file.rfile.bcfile;
+
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile;
+import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile.BlockRead;
+import org.apache.accumulo.core.file.rfile.bcfile.CompareUtils.Scalar;
+import org.apache.accumulo.core.file.rfile.bcfile.CompareUtils.ScalarComparator;
+import org.apache.accumulo.core.file.rfile.bcfile.CompareUtils.ScalarLong;
+import org.apache.accumulo.core.file.rfile.bcfile.Compression.Algorithm;
+import org.apache.accumulo.core.file.rfile.bcfile.Utils.Version;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.compress.Compressor;
+import org.apache.hadoop.io.compress.Decompressor;
+
+/**
+ * Block Compressed file, the underlying physical storage layer for TFile. BCFile provides the basic block level compression for the data block and meta blocks.
+ * It is separated from TFile as it may be used for other block-compressed file implementation.
+ */
+public final class BCFile {
+ // the current version of BCFile impl, increment them (major or minor) made
+ // enough changes
+ static final Version API_VERSION = new Version((short) 1, (short) 0);
+ static final Log LOG = LogFactory.getLog(BCFile.class);
+
+ /**
+ * Prevent the instantiation of BCFile objects.
+ */
+ private BCFile() {
+ // nothing
+ }
+
+ /**
+ * BCFile writer, the entry point for creating a new BCFile.
+ */
+ static public class Writer implements Closeable {
+ private final FSDataOutputStream out;
+ private final Configuration conf;
+ // the single meta block containing index of compressed data blocks
+ final DataIndex dataIndex;
+ // index for meta blocks
+ final MetaIndex metaIndex;
+ boolean blkInProgress = false;
+ private boolean metaBlkSeen = false;
+ private boolean closed = false;
+ long errorCount = 0;
+ // reusable buffers.
+ private BytesWritable fsOutputBuffer;
+
+ /**
+ * Call-back interface to register a block after a block is closed.
+ */
+ private static interface BlockRegister {
+ /**
+ * Register a block that is fully closed.
+ *
+ * @param raw
+ * The size of block in terms of uncompressed bytes.
+ * @param offsetStart
+ * The start offset of the block.
+ * @param offsetEnd
+ * One byte after the end of the block. Compressed block size is offsetEnd - offsetStart.
+ */
+ public void register(long raw, long offsetStart, long offsetEnd);
+ }
+
+ /**
+ * Intermediate class that maintain the state of a Writable Compression Block.
+ */
+ private static final class WBlockState {
+ private final Algorithm compressAlgo;
+ private Compressor compressor; // !null only if using native
+ // Hadoop compression
+ private final FSDataOutputStream fsOut;
+ private final long posStart;
+ private final SimpleBufferedOutputStream fsBufferedOutput;
+ private OutputStream out;
+
+ /**
+ * @param compressionAlgo
+ * The compression algorithm to be used to for compression.
- * @throws IOException
+ */
+ public WBlockState(Algorithm compressionAlgo, FSDataOutputStream fsOut, BytesWritable fsOutputBuffer, Configuration conf) throws IOException {
+ this.compressAlgo = compressionAlgo;
+ this.fsOut = fsOut;
+ this.posStart = fsOut.getPos();
+
+ fsOutputBuffer.setCapacity(TFile.getFSOutputBufferSize(conf));
+
+ this.fsBufferedOutput = new SimpleBufferedOutputStream(this.fsOut, fsOutputBuffer.getBytes());
+ this.compressor = compressAlgo.getCompressor();
+
+ try {
+ this.out = compressionAlgo.createCompressionStream(fsBufferedOutput, compressor, 0);
+ } catch (IOException e) {
+ compressAlgo.returnCompressor(compressor);
+ throw e;
+ }
+ }
+
+ /**
+ * Get the output stream for BlockAppender's consumption.
+ *
+ * @return the output stream suitable for writing block data.
+ */
+ OutputStream getOutputStream() {
+ return out;
+ }
+
+ /**
+ * Get the current position in file.
+ *
+ * @return The current byte offset in underlying file.
+ * @throws IOException
+ */
+ long getCurrentPos() throws IOException {
+ return fsOut.getPos() + fsBufferedOutput.size();
+ }
+
+ long getStartPos() {
+ return posStart;
+ }
+
+ /**
+ * Current size of compressed data.
+ *
+ * @throws IOException
+ */
+ long getCompressedSize() throws IOException {
+ long ret = getCurrentPos() - posStart;
+ return ret;
+ }
+
+ /**
+ * Finishing up the current block.
+ */
+ public void finish() throws IOException {
+ try {
+ if (out != null) {
+ out.flush();
+ out = null;
+ }
+ } finally {
+ compressAlgo.returnCompressor(compressor);
+ compressor = null;
+ }
+ }
+ }
+
+ /**
+ * Access point to stuff data into a block.
+ *
+ */
+ public class BlockAppender extends DataOutputStream {
+ private final BlockRegister blockRegister;
+ private final WBlockState wBlkState;
+ private boolean closed = false;
+
+ /**
+ * Constructor
+ *
+ * @param register
+ * the block register, which is called when the block is closed.
+ * @param wbs
+ * The writable compression block state.
+ */
+ BlockAppender(BlockRegister register, WBlockState wbs) {
+ super(wbs.getOutputStream());
+ this.blockRegister = register;
+ this.wBlkState = wbs;
+ }
+
+ /**
+ * Get the raw size of the block.
+ *
+ * @return the number of uncompressed bytes written through the BlockAppender so far.
- * @throws IOException
+ */
+ public long getRawSize() throws IOException {
+ /**
+ * Expecting the size() of a block not exceeding 4GB. Assuming the size() will wrap to negative integer if it exceeds 2GB.
+ */
+ return size() & 0x00000000ffffffffL;
+ }
+
+ /**
+ * Get the compressed size of the block in progress.
+ *
+ * @return the number of compressed bytes written to the underlying FS file. The size may be smaller than actual need to compress the all data written due
+ * to internal buffering inside the compressor.
- * @throws IOException
+ */
+ public long getCompressedSize() throws IOException {
+ return wBlkState.getCompressedSize();
+ }
+
+ public long getStartPos() {
+ return wBlkState.getStartPos();
+ }
+
+ @Override
+ public void flush() {
+ // The down stream is a special kind of stream that finishes a
+ // compression block upon flush. So we disable flush() here.
+ }
+
+ /**
+ * Signaling the end of write to the block. The block register will be called for registering the finished block.
+ */
+ @Override
+ public void close() throws IOException {
+ if (closed == true) {
+ return;
+ }
+ try {
+ ++errorCount;
+ wBlkState.finish();
+ blockRegister.register(getRawSize(), wBlkState.getStartPos(), wBlkState.getCurrentPos());
+ --errorCount;
+ } finally {
+ closed = true;
+ blkInProgress = false;
+ }
+ }
+ }
+
+ /**
+ * Constructor
+ *
+ * @param fout
+ * FS output stream.
+ * @param compressionName
+ * Name of the compression algorithm, which will be used for all data blocks.
- * @throws IOException
+ * @see Compression#getSupportedAlgorithms
+ */
+ public Writer(FSDataOutputStream fout, String compressionName, Configuration conf, boolean trackDataBlocks) throws IOException {
+ if (fout.getPos() != 0) {
+ throw new IOException("Output file not at zero offset.");
+ }
+
+ this.out = fout;
+ this.conf = conf;
+ dataIndex = new DataIndex(compressionName, trackDataBlocks);
+ metaIndex = new MetaIndex();
+ fsOutputBuffer = new BytesWritable();
+ Magic.write(fout);
+ }
+
+ /**
+ * Close the BCFile Writer. Attempting to use the Writer after calling <code>close</code> is not allowed and may lead to undetermined results.
+ */
++ @Override
+ public void close() throws IOException {
+ if (closed == true) {
+ return;
+ }
+
+ try {
+ if (errorCount == 0) {
+ if (blkInProgress == true) {
+ throw new IllegalStateException("Close() called with active block appender.");
+ }
+
+ // add metaBCFileIndex to metaIndex as the last meta block
+ BlockAppender appender = prepareMetaBlock(DataIndex.BLOCK_NAME, getDefaultCompressionAlgorithm());
+ try {
+ dataIndex.write(appender);
+ } finally {
+ appender.close();
+ }
+
+ long offsetIndexMeta = out.getPos();
+ metaIndex.write(out);
+
+ // Meta Index and the trailing section are written out directly.
+ out.writeLong(offsetIndexMeta);
+
+ API_VERSION.write(out);
+ Magic.write(out);
+ out.flush();
+ }
+ } finally {
+ closed = true;
+ }
+ }
+
+ private Algorithm getDefaultCompressionAlgorithm() {
+ return dataIndex.getDefaultCompressionAlgorithm();
+ }
+
+ private BlockAppender prepareMetaBlock(String name, Algorithm compressAlgo) throws IOException, MetaBlockAlreadyExists {
+ if (blkInProgress == true) {
+ throw new IllegalStateException("Cannot create Meta Block until previous block is closed.");
+ }
+
+ if (metaIndex.getMetaByName(name) != null) {
+ throw new MetaBlockAlreadyExists("name=" + name);
+ }
+
+ MetaBlockRegister mbr = new MetaBlockRegister(name, compressAlgo);
+ WBlockState wbs = new WBlockState(compressAlgo, out, fsOutputBuffer, conf);
+ BlockAppender ba = new BlockAppender(mbr, wbs);
+ blkInProgress = true;
+ metaBlkSeen = true;
+ return ba;
+ }
+
+ /**
+ * Create a Meta Block and obtain an output stream for adding data into the block. There can only be one BlockAppender stream active at any time. Regular
+ * Blocks may not be created after the first Meta Blocks. The caller must call BlockAppender.close() to conclude the block creation.
+ *
+ * @param name
+ * The name of the Meta Block. The name must not conflict with existing Meta Blocks.
+ * @param compressionName
+ * The name of the compression algorithm to be used.
+ * @return The BlockAppender stream
- * @throws IOException
+ * @throws MetaBlockAlreadyExists
+ * If the meta block with the name already exists.
+ */
+ public BlockAppender prepareMetaBlock(String name, String compressionName) throws IOException, MetaBlockAlreadyExists {
+ return prepareMetaBlock(name, Compression.getCompressionAlgorithmByName(compressionName));
+ }
+
+ /**
+ * Create a Meta Block and obtain an output stream for adding data into the block. The Meta Block will be compressed with the same compression algorithm as
+ * data blocks. There can only be one BlockAppender stream active at any time. Regular Blocks may not be created after the first Meta Blocks. The caller
+ * must call BlockAppender.close() to conclude the block creation.
+ *
+ * @param name
+ * The name of the Meta Block. The name must not conflict with existing Meta Blocks.
+ * @return The BlockAppender stream
+ * @throws MetaBlockAlreadyExists
+ * If the meta block with the name already exists.
- * @throws IOException
+ */
+ public BlockAppender prepareMetaBlock(String name) throws IOException, MetaBlockAlreadyExists {
+ return prepareMetaBlock(name, getDefaultCompressionAlgorithm());
+ }
+
+ /**
+ * Create a Data Block and obtain an output stream for adding data into the block. There can only be one BlockAppender stream active at any time. Data
+ * Blocks may not be created after the first Meta Blocks. The caller must call BlockAppender.close() to conclude the block creation.
+ *
+ * @return The BlockAppender stream
- * @throws IOException
+ */
+ public BlockAppender prepareDataBlock() throws IOException {
+ if (blkInProgress == true) {
+ throw new IllegalStateException("Cannot create Data Block until previous block is closed.");
+ }
+
+ if (metaBlkSeen == true) {
+ throw new IllegalStateException("Cannot create Data Block after Meta Blocks.");
+ }
+
+ DataBlockRegister dbr = new DataBlockRegister();
+
+ WBlockState wbs = new WBlockState(getDefaultCompressionAlgorithm(), out, fsOutputBuffer, conf);
+ BlockAppender ba = new BlockAppender(dbr, wbs);
+ blkInProgress = true;
+ return ba;
+ }
+
+ /**
+ * Callback to make sure a meta block is added to the internal list when its stream is closed.
+ */
+ private class MetaBlockRegister implements BlockRegister {
+ private final String name;
+ private final Algorithm compressAlgo;
+
+ MetaBlockRegister(String name, Algorithm compressAlgo) {
+ this.name = name;
+ this.compressAlgo = compressAlgo;
+ }
+
++ @Override
+ public void register(long raw, long begin, long end) {
+ metaIndex.addEntry(new MetaIndexEntry(name, compressAlgo, new BlockRegion(begin, end - begin, raw)));
+ }
+ }
+
+ /**
+ * Callback to make sure a data block is added to the internal list when it's being closed.
+ *
+ */
+ private class DataBlockRegister implements BlockRegister {
+ DataBlockRegister() {
+ // do nothing
+ }
+
++ @Override
+ public void register(long raw, long begin, long end) {
+ dataIndex.addBlockRegion(new BlockRegion(begin, end - begin, raw));
+ }
+ }
+ }
+
+ /**
+ * BCFile Reader, interface to read the file's data and meta blocks.
+ */
+ static public class Reader implements Closeable {
+ private static final String META_NAME = "BCFile.metaindex";
+ private final FSDataInputStream in;
+ private final Configuration conf;
+ final DataIndex dataIndex;
+ // Index for meta blocks
+ final MetaIndex metaIndex;
+ final Version version;
+
+ /**
+ * Intermediate class that maintain the state of a Readable Compression Block.
+ */
+ static private final class RBlockState {
+ private final Algorithm compressAlgo;
+ private Decompressor decompressor;
+ private final BlockRegion region;
+ private final InputStream in;
+
+ public RBlockState(Algorithm compressionAlgo, FSDataInputStream fsin, BlockRegion region, Configuration conf) throws IOException {
+ this.compressAlgo = compressionAlgo;
+ this.region = region;
+ this.decompressor = compressionAlgo.getDecompressor();
+
+ try {
+ this.in = compressAlgo.createDecompressionStream(new BoundedRangeFileInputStream(fsin, this.region.getOffset(), this.region.getCompressedSize()),
+ decompressor, TFile.getFSInputBufferSize(conf));
+ } catch (IOException e) {
+ compressAlgo.returnDecompressor(decompressor);
+ throw e;
+ }
+ }
+
+ /**
+ * Get the output stream for BlockAppender's consumption.
+ *
+ * @return the output stream suitable for writing block data.
+ */
+ public InputStream getInputStream() {
+ return in;
+ }
+
+ public String getCompressionName() {
+ return compressAlgo.getName();
+ }
+
+ public BlockRegion getBlockRegion() {
+ return region;
+ }
+
+ public void finish() throws IOException {
+ try {
+ in.close();
+ } finally {
+ compressAlgo.returnDecompressor(decompressor);
+ decompressor = null;
+ }
+ }
+ }
+
+ /**
+ * Access point to read a block.
+ */
+ public static class BlockReader extends DataInputStream {
+ private final RBlockState rBlkState;
+ private boolean closed = false;
+
+ BlockReader(RBlockState rbs) {
+ super(rbs.getInputStream());
+ rBlkState = rbs;
+ }
+
+ /**
+ * Finishing reading the block. Release all resources.
+ */
+ @Override
+ public void close() throws IOException {
+ if (closed == true) {
+ return;
+ }
+ try {
+ // Do not set rBlkState to null. People may access stats after calling
+ // close().
+ rBlkState.finish();
+ } finally {
+ closed = true;
+ }
+ }
+
+ /**
+ * Get the name of the compression algorithm used to compress the block.
+ *
+ * @return name of the compression algorithm.
+ */
+ public String getCompressionName() {
+ return rBlkState.getCompressionName();
+ }
+
+ /**
+ * Get the uncompressed size of the block.
+ *
+ * @return uncompressed size of the block.
+ */
+ public long getRawSize() {
+ return rBlkState.getBlockRegion().getRawSize();
+ }
+
+ /**
+ * Get the compressed size of the block.
+ *
+ * @return compressed size of the block.
+ */
+ public long getCompressedSize() {
+ return rBlkState.getBlockRegion().getCompressedSize();
+ }
+
+ /**
+ * Get the starting position of the block in the file.
+ *
+ * @return the starting position of the block in the file.
+ */
+ public long getStartPos() {
+ return rBlkState.getBlockRegion().getOffset();
+ }
+ }
+
+ /**
+ * Constructor
+ *
+ * @param fin
+ * FS input stream.
+ * @param fileLength
+ * Length of the corresponding file
- * @throws IOException
+ */
+ public Reader(FSDataInputStream fin, long fileLength, Configuration conf) throws IOException {
+ this.in = fin;
+ this.conf = conf;
+
+ // move the cursor to the beginning of the tail, containing: offset to the
+ // meta block index, version and magic
+ fin.seek(fileLength - Magic.size() - Version.size() - Long.SIZE / Byte.SIZE);
+ long offsetIndexMeta = fin.readLong();
+ version = new Version(fin);
+ Magic.readAndVerify(fin);
+
+ if (!version.compatibleWith(BCFile.API_VERSION)) {
+ throw new RuntimeException("Incompatible BCFile fileBCFileVersion.");
+ }
+
+ // read meta index
+ fin.seek(offsetIndexMeta);
+ metaIndex = new MetaIndex(fin);
+
+ // read data:BCFile.index, the data block index
+ BlockReader blockR = getMetaBlock(DataIndex.BLOCK_NAME);
+ try {
+ dataIndex = new DataIndex(blockR);
+ } finally {
+ blockR.close();
+ }
+ }
+
+ public Reader(CachableBlockFile.Reader cache, FSDataInputStream fin, long fileLength, Configuration conf) throws IOException {
+ this.in = fin;
+ this.conf = conf;
+
+ BlockRead cachedMetaIndex = cache.getCachedMetaBlock(META_NAME);
+ BlockRead cachedDataIndex = cache.getCachedMetaBlock(DataIndex.BLOCK_NAME);
+
+ if (cachedMetaIndex == null || cachedDataIndex == null) {
+ // move the cursor to the beginning of the tail, containing: offset to the
+ // meta block index, version and magic
+ fin.seek(fileLength - Magic.size() - Version.size() - Long.SIZE / Byte.SIZE);
+ long offsetIndexMeta = fin.readLong();
+ version = new Version(fin);
+ Magic.readAndVerify(fin);
+
+ if (!version.compatibleWith(BCFile.API_VERSION)) {
+ throw new RuntimeException("Incompatible BCFile fileBCFileVersion.");
+ }
+
+ // read meta index
+ fin.seek(offsetIndexMeta);
+ metaIndex = new MetaIndex(fin);
+ if (cachedMetaIndex == null) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ metaIndex.write(dos);
+ dos.close();
+ cache.cacheMetaBlock(META_NAME, baos.toByteArray());
+ }
+
+ // read data:BCFile.index, the data block index
+ if (cachedDataIndex == null) {
+ BlockReader blockR = getMetaBlock(DataIndex.BLOCK_NAME);
+ cachedDataIndex = cache.cacheMetaBlock(DataIndex.BLOCK_NAME, blockR);
+ }
+
+ dataIndex = new DataIndex(cachedDataIndex);
+ cachedDataIndex.close();
+
+ } else {
+ // Logger.getLogger(Reader.class).debug("Read bcfile !METADATA from cache");
+ version = null;
+ metaIndex = new MetaIndex(cachedMetaIndex);
+ dataIndex = new DataIndex(cachedDataIndex);
+ }
+ }
+
+ /**
+ * Get the name of the default compression algorithm.
+ *
+ * @return the name of the default compression algorithm.
+ */
+ public String getDefaultCompressionName() {
+ return dataIndex.getDefaultCompressionAlgorithm().getName();
+ }
+
+ /**
+ * Get version of BCFile file being read.
+ *
+ * @return version of BCFile file being read.
+ */
+ public Version getBCFileVersion() {
+ return version;
+ }
+
+ /**
+ * Get version of BCFile API.
+ *
+ * @return version of BCFile API.
+ */
+ public Version getAPIVersion() {
+ return API_VERSION;
+ }
+
+ /**
+ * Finishing reading the BCFile. Release all resources.
+ */
++ @Override
+ public void close() {
+ // nothing to be done now
+ }
+
+ /**
+ * Get the number of data blocks.
+ *
+ * @return the number of data blocks.
+ */
+ public int getBlockCount() {
+ return dataIndex.getBlockRegionList().size();
+ }
+
+ /**
+ * Stream access to a Meta Block.
+ *
+ * @param name
+ * meta block name
+ * @return BlockReader input stream for reading the meta block.
- * @throws IOException
+ * @throws MetaBlockDoesNotExist
+ * The Meta Block with the given name does not exist.
+ */
+ public BlockReader getMetaBlock(String name) throws IOException, MetaBlockDoesNotExist {
+ MetaIndexEntry imeBCIndex = metaIndex.getMetaByName(name);
+ if (imeBCIndex == null) {
+ throw new MetaBlockDoesNotExist("name=" + name);
+ }
+
+ BlockRegion region = imeBCIndex.getRegion();
+ return createReader(imeBCIndex.getCompressionAlgorithm(), region);
+ }
+
+ /**
+ * Stream access to a Data Block.
+ *
+ * @param blockIndex
+ * 0-based data block index.
+ * @return BlockReader input stream for reading the data block.
- * @throws IOException
+ */
+ public BlockReader getDataBlock(int blockIndex) throws IOException {
+ if (blockIndex < 0 || blockIndex >= getBlockCount()) {
+ throw new IndexOutOfBoundsException(String.format("blockIndex=%d, numBlocks=%d", blockIndex, getBlockCount()));
+ }
+
+ BlockRegion region = dataIndex.getBlockRegionList().get(blockIndex);
+ return createReader(dataIndex.getDefaultCompressionAlgorithm(), region);
+ }
+
+ public BlockReader getDataBlock(long offset, long compressedSize, long rawSize) throws IOException {
+ BlockRegion region = new BlockRegion(offset, compressedSize, rawSize);
+ return createReader(dataIndex.getDefaultCompressionAlgorithm(), region);
+ }
+
+ private BlockReader createReader(Algorithm compressAlgo, BlockRegion region) throws IOException {
+ RBlockState rbs = new RBlockState(compressAlgo, in, region, conf);
+ return new BlockReader(rbs);
+ }
+
+ /**
+ * Find the smallest Block index whose starting offset is greater than or equal to the specified offset.
+ *
+ * @param offset
+ * User-specific offset.
+ * @return the index to the data Block if such block exists; or -1 otherwise.
+ */
+ public int getBlockIndexNear(long offset) {
+ ArrayList<BlockRegion> list = dataIndex.getBlockRegionList();
+ int idx = Utils.lowerBound(list, new ScalarLong(offset), new ScalarComparator());
+
+ if (idx == list.size()) {
+ return -1;
+ }
+
+ return idx;
+ }
+ }
+
+ /**
+ * Index for all Meta blocks.
+ */
+ static class MetaIndex {
+ // use a tree map, for getting a meta block entry by name
+ final Map<String,MetaIndexEntry> index;
+
+ // for write
+ public MetaIndex() {
+ index = new TreeMap<String,MetaIndexEntry>();
+ }
+
+ // for read, construct the map from the file
+ public MetaIndex(DataInput in) throws IOException {
+ int count = Utils.readVInt(in);
+ index = new TreeMap<String,MetaIndexEntry>();
+
+ for (int nx = 0; nx < count; nx++) {
+ MetaIndexEntry indexEntry = new MetaIndexEntry(in);
+ index.put(indexEntry.getMetaName(), indexEntry);
+ }
+ }
+
+ public void addEntry(MetaIndexEntry indexEntry) {
+ index.put(indexEntry.getMetaName(), indexEntry);
+ }
+
+ public MetaIndexEntry getMetaByName(String name) {
+ return index.get(name);
+ }
+
+ public void write(DataOutput out) throws IOException {
+ Utils.writeVInt(out, index.size());
+
+ for (MetaIndexEntry indexEntry : index.values()) {
+ indexEntry.write(out);
+ }
+ }
+ }
+
+ /**
+ * An entry describes a meta block in the MetaIndex.
+ */
+ static final class MetaIndexEntry {
+ private final String metaName;
+ private final Algorithm compressionAlgorithm;
+ private final static String defaultPrefix = "data:";
+
+ private final BlockRegion region;
+
+ public MetaIndexEntry(DataInput in) throws IOException {
+ String fullMetaName = Utils.readString(in);
+ if (fullMetaName.startsWith(defaultPrefix)) {
+ metaName = fullMetaName.substring(defaultPrefix.length(), fullMetaName.length());
+ } else {
+ throw new IOException("Corrupted Meta region Index");
+ }
+
+ compressionAlgorithm = Compression.getCompressionAlgorithmByName(Utils.readString(in));
+ region = new BlockRegion(in);
+ }
+
+ public MetaIndexEntry(String metaName, Algorithm compressionAlgorithm, BlockRegion region) {
+ this.metaName = metaName;
+ this.compressionAlgorithm = compressionAlgorithm;
+ this.region = region;
+ }
+
+ public String getMetaName() {
+ return metaName;
+ }
+
+ public Algorithm getCompressionAlgorithm() {
+ return compressionAlgorithm;
+ }
+
+ public BlockRegion getRegion() {
+ return region;
+ }
+
+ public void write(DataOutput out) throws IOException {
+ Utils.writeString(out, defaultPrefix + metaName);
+ Utils.writeString(out, compressionAlgorithm.getName());
+
+ region.write(out);
+ }
+ }
+
+ /**
+ * Index of all compressed data blocks.
+ */
+ static class DataIndex {
+ final static String BLOCK_NAME = "BCFile.index";
+
+ private final Algorithm defaultCompressionAlgorithm;
+
+ // for data blocks, each entry specifies a block's offset, compressed size
+ // and raw size
+ private final ArrayList<BlockRegion> listRegions;
+
+ private boolean trackBlocks;
+
+ // for read, deserialized from a file
+ public DataIndex(DataInput in) throws IOException {
+ defaultCompressionAlgorithm = Compression.getCompressionAlgorithmByName(Utils.readString(in));
+
+ int n = Utils.readVInt(in);
+ listRegions = new ArrayList<BlockRegion>(n);
+
+ for (int i = 0; i < n; i++) {
+ BlockRegion region = new BlockRegion(in);
+ listRegions.add(region);
+ }
+ }
+
+ // for write
+ public DataIndex(String defaultCompressionAlgorithmName, boolean trackBlocks) {
+ this.trackBlocks = trackBlocks;
+ this.defaultCompressionAlgorithm = Compression.getCompressionAlgorithmByName(defaultCompressionAlgorithmName);
+ listRegions = new ArrayList<BlockRegion>();
+ }
+
+ public Algorithm getDefaultCompressionAlgorithm() {
+ return defaultCompressionAlgorithm;
+ }
+
+ public ArrayList<BlockRegion> getBlockRegionList() {
+ return listRegions;
+ }
+
+ public void addBlockRegion(BlockRegion region) {
+ if (trackBlocks)
+ listRegions.add(region);
+ }
+
+ public void write(DataOutput out) throws IOException {
+ Utils.writeString(out, defaultCompressionAlgorithm.getName());
+
+ Utils.writeVInt(out, listRegions.size());
+
+ for (BlockRegion region : listRegions) {
+ region.write(out);
+ }
+ }
+ }
+
+ /**
+ * Magic number uniquely identifying a BCFile in the header/footer.
+ */
+ static final class Magic {
+ private final static byte[] AB_MAGIC_BCFILE = {
+ // ... total of 16 bytes
+ (byte) 0xd1, (byte) 0x11, (byte) 0xd3, (byte) 0x68, (byte) 0x91, (byte) 0xb5, (byte) 0xd7, (byte) 0xb6, (byte) 0x39, (byte) 0xdf, (byte) 0x41,
+ (byte) 0x40, (byte) 0x92, (byte) 0xba, (byte) 0xe1, (byte) 0x50};
+
+ public static void readAndVerify(DataInput in) throws IOException {
+ byte[] abMagic = new byte[size()];
+ in.readFully(abMagic);
+
+ // check against AB_MAGIC_BCFILE, if not matching, throw an
+ // Exception
+ if (!Arrays.equals(abMagic, AB_MAGIC_BCFILE)) {
+ throw new IOException("Not a valid BCFile.");
+ }
+ }
+
+ public static void write(DataOutput out) throws IOException {
+ out.write(AB_MAGIC_BCFILE);
+ }
+
+ public static int size() {
+ return AB_MAGIC_BCFILE.length;
+ }
+ }
+
+ /**
+ * Block region.
+ */
+ static final class BlockRegion implements Scalar {
+ private final long offset;
+ private final long compressedSize;
+ private final long rawSize;
+
+ public BlockRegion(DataInput in) throws IOException {
+ offset = Utils.readVLong(in);
+ compressedSize = Utils.readVLong(in);
+ rawSize = Utils.readVLong(in);
+ }
+
+ public BlockRegion(long offset, long compressedSize, long rawSize) {
+ this.offset = offset;
+ this.compressedSize = compressedSize;
+ this.rawSize = rawSize;
+ }
+
+ public void write(DataOutput out) throws IOException {
+ Utils.writeVLong(out, offset);
+ Utils.writeVLong(out, compressedSize);
+ Utils.writeVLong(out, rawSize);
+ }
+
+ public long getOffset() {
+ return offset;
+ }
+
+ public long getCompressedSize() {
+ return compressedSize;
+ }
+
+ public long getRawSize() {
+ return rawSize;
+ }
+
+ @Override
+ public long magnitude() {
+ return offset;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/ByteArray.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/ByteArray.java
index 2b57638,0000000..d7734a2
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/ByteArray.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/ByteArray.java
@@@ -1,91 -1,0 +1,89 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.accumulo.core.file.rfile.bcfile;
+
+import org.apache.hadoop.io.BytesWritable;
+
+/**
+ * Adaptor class to wrap byte-array backed objects (including java byte array) as RawComparable objects.
+ */
+public final class ByteArray implements RawComparable {
+ private final byte[] buffer;
+ private final int offset;
+ private final int len;
+
+ /**
+ * Constructing a ByteArray from a {@link BytesWritable}.
- *
- * @param other
+ */
+ public ByteArray(BytesWritable other) {
+ this(other.getBytes(), 0, other.getLength());
+ }
+
+ /**
+ * Wrap a whole byte array as a RawComparable.
+ *
+ * @param buffer
+ * the byte array buffer.
+ */
+ public ByteArray(byte[] buffer) {
+ this(buffer, 0, buffer.length);
+ }
+
+ /**
+ * Wrap a partial byte array as a RawComparable.
+ *
+ * @param buffer
+ * the byte array buffer.
+ * @param offset
+ * the starting offset
+ * @param len
+ * the length of the consecutive bytes to be wrapped.
+ */
+ public ByteArray(byte[] buffer, int offset, int len) {
+ if ((offset | len | (buffer.length - offset - len)) < 0) {
+ throw new IndexOutOfBoundsException();
+ }
+ this.buffer = buffer;
+ this.offset = offset;
+ this.len = len;
+ }
+
+ /**
+ * @return the underlying buffer.
+ */
+ @Override
+ public byte[] buffer() {
+ return buffer;
+ }
+
+ /**
+ * @return the offset in the buffer.
+ */
+ @Override
+ public int offset() {
+ return offset;
+ }
+
+ /**
+ * @return the size of the byte array.
+ */
+ @Override
+ public int size() {
+ return len;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Chunk.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Chunk.java
index a075d87,0000000..345d406
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Chunk.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Chunk.java
@@@ -1,418 -1,0 +1,416 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.accumulo.core.file.rfile.bcfile;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Several related classes to support chunk-encoded sub-streams on top of a regular stream.
+ */
+final class Chunk {
+
+ /**
+ * Prevent the instantiation of class.
+ */
+ private Chunk() {
+ // nothing
+ }
+
+ /**
+ * Decoding a chain of chunks encoded through ChunkEncoder or SingleChunkEncoder.
+ */
+ static public class ChunkDecoder extends InputStream {
+ private DataInputStream in = null;
+ private boolean lastChunk;
+ private int remain = 0;
+ private boolean closed;
+
+ public ChunkDecoder() {
+ lastChunk = true;
+ closed = true;
+ }
+
+ public void reset(DataInputStream downStream) {
+ // no need to wind forward the old input.
+ in = downStream;
+ lastChunk = false;
+ remain = 0;
+ closed = false;
+ }
+
+ /**
+ * Constructor
+ *
+ * @param in
+ * The source input stream which contains chunk-encoded data stream.
+ */
+ public ChunkDecoder(DataInputStream in) {
+ this.in = in;
+ lastChunk = false;
+ closed = false;
+ }
+
+ /**
+ * Have we reached the last chunk.
+ *
+ * @return true if we have reached the last chunk.
- * @throws java.io.IOException
+ */
+ public boolean isLastChunk() throws IOException {
+ checkEOF();
+ return lastChunk;
+ }
+
+ /**
+ * How many bytes remain in the current chunk?
+ *
+ * @return remaining bytes left in the current chunk.
- * @throws java.io.IOException
+ */
+ public int getRemain() throws IOException {
+ checkEOF();
+ return remain;
+ }
+
+ /**
+ * Reading the length of next chunk.
+ *
+ * @throws java.io.IOException
+ * when no more data is available.
+ */
+ private void readLength() throws IOException {
+ remain = Utils.readVInt(in);
+ if (remain >= 0) {
+ lastChunk = true;
+ } else {
+ remain = -remain;
+ }
+ }
+
+ /**
+ * Check whether we reach the end of the stream.
+ *
+ * @return false if the chunk encoded stream has more data to read (in which case available() will be greater than 0); true otherwise.
+ * @throws java.io.IOException
+ * on I/O errors.
+ */
+ private boolean checkEOF() throws IOException {
+ if (isClosed())
+ return true;
+ while (true) {
+ if (remain > 0)
+ return false;
+ if (lastChunk)
+ return true;
+ readLength();
+ }
+ }
+
+ @Override
+ /*
+ * This method never blocks the caller. Returning 0 does not mean we reach the end of the stream.
+ */
+ public int available() {
+ return remain;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (checkEOF())
+ return -1;
+ int ret = in.read();
+ if (ret < 0)
+ throw new IOException("Corrupted chunk encoding stream");
+ --remain;
+ return ret;
+ }
+
+ @Override
+ public int read(byte[] b) throws IOException {
+ return read(b, 0, b.length);
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ if ((off | len | (off + len) | (b.length - (off + len))) < 0) {
+ throw new IndexOutOfBoundsException();
+ }
+
+ if (!checkEOF()) {
+ int n = Math.min(remain, len);
+ int ret = in.read(b, off, n);
+ if (ret < 0)
+ throw new IOException("Corrupted chunk encoding stream");
+ remain -= ret;
+ return ret;
+ }
+ return -1;
+ }
+
+ @Override
+ public long skip(long n) throws IOException {
+ if (!checkEOF()) {
+ long ret = in.skip(Math.min(remain, n));
+ remain -= ret;
+ return ret;
+ }
+ return 0;
+ }
+
+ @Override
+ public boolean markSupported() {
+ return false;
+ }
+
+ public boolean isClosed() {
+ return closed;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed == false) {
+ try {
+ while (!checkEOF()) {
+ skip(Integer.MAX_VALUE);
+ }
+ } finally {
+ closed = true;
+ }
+ }
+ }
+ }
+
+ /**
+ * Chunk Encoder. Encoding the output data into a chain of chunks in the following sequences: -len1, byte[len1], -len2, byte[len2], ... len_n, byte[len_n].
+ * Where len1, len2, ..., len_n are the lengths of the data chunks. Non-terminal chunks have their lengths negated. Non-terminal chunks cannot have length 0.
+ * All lengths are in the range of 0 to Integer.MAX_VALUE and are encoded in Utils.VInt format.
+ */
+ static public class ChunkEncoder extends OutputStream {
+ /**
+ * The data output stream it connects to.
+ */
+ private DataOutputStream out;
+
+ /**
+ * The internal buffer that is only used when we do not know the advertised size.
+ */
+ private byte buf[];
+
+ /**
+ * The number of valid bytes in the buffer. This value is always in the range <tt>0</tt> through <tt>buf.length</tt>; elements <tt>buf[0]</tt> through
+ * <tt>buf[count-1]</tt> contain valid byte data.
+ */
+ private int count;
+
+ /**
+ * Constructor.
+ *
+ * @param out
+ * the underlying output stream.
+ * @param buf
+ * user-supplied buffer. The buffer would be used exclusively by the ChunkEncoder during its life cycle.
+ */
+ public ChunkEncoder(DataOutputStream out, byte[] buf) {
+ this.out = out;
+ this.buf = buf;
+ this.count = 0;
+ }
+
+ /**
+ * Write out a chunk.
+ *
+ * @param chunk
+ * The chunk buffer.
+ * @param offset
+ * Offset to chunk buffer for the beginning of chunk.
+ * @param len
+ * @param last
+ * Is this the last call to flushBuffer?
+ */
+ private void writeChunk(byte[] chunk, int offset, int len, boolean last) throws IOException {
+ if (last) { // always write out the length for the last chunk.
+ Utils.writeVInt(out, len);
+ if (len > 0) {
+ out.write(chunk, offset, len);
+ }
+ } else {
+ if (len > 0) {
+ Utils.writeVInt(out, -len);
+ out.write(chunk, offset, len);
+ }
+ }
+ }
+
+ /**
+ * Write out a chunk that is a concatenation of the internal buffer plus user supplied data. This will never be the last block.
+ *
+ * @param data
+ * User supplied data buffer.
+ * @param offset
+ * Offset to user data buffer.
+ * @param len
+ * User data buffer size.
+ */
+ private void writeBufData(byte[] data, int offset, int len) throws IOException {
+ if (count + len > 0) {
+ Utils.writeVInt(out, -(count + len));
+ out.write(buf, 0, count);
+ count = 0;
+ out.write(data, offset, len);
+ }
+ }
+
+ /**
+ * Flush the internal buffer.
+ *
+ * Is this the last call to flushBuffer?
+ *
+ * @throws java.io.IOException
+ */
+ private void flushBuffer() throws IOException {
+ if (count > 0) {
+ writeChunk(buf, 0, count, false);
+ count = 0;
+ }
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ if (count >= buf.length) {
+ flushBuffer();
+ }
+ buf[count++] = (byte) b;
+ }
+
+ @Override
+ public void write(byte b[]) throws IOException {
+ write(b, 0, b.length);
+ }
+
+ @Override
+ public void write(byte b[], int off, int len) throws IOException {
+ if ((len + count) >= buf.length) {
+ /*
+ * If the input data do not fit in buffer, flush the output buffer and then write the data directly. In this way buffered streams will cascade
+ * harmlessly.
+ */
+ writeBufData(b, off, len);
+ return;
+ }
+
+ System.arraycopy(b, off, buf, count, len);
+ count += len;
+ }
+
+ @Override
+ public void flush() throws IOException {
+ flushBuffer();
+ out.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (buf != null) {
+ try {
+ writeChunk(buf, 0, count, true);
+ } finally {
+ buf = null;
+ out = null;
+ }
+ }
+ }
+ }
+
+ /**
+ * Encode the whole stream as a single chunk. Expecting to know the size of the chunk up-front.
+ */
+ static public class SingleChunkEncoder extends OutputStream {
+ /**
+ * The data output stream it connects to.
+ */
+ private final DataOutputStream out;
+
+ /**
+ * The remaining bytes to be written.
+ */
+ private int remain;
+ private boolean closed = false;
+
+ /**
+ * Constructor.
+ *
+ * @param out
+ * the underlying output stream.
+ * @param size
+ * The total # of bytes to be written as a single chunk.
+ * @throws java.io.IOException
+ * if an I/O error occurs.
+ */
+ public SingleChunkEncoder(DataOutputStream out, int size) throws IOException {
+ this.out = out;
+ this.remain = size;
+ Utils.writeVInt(out, size);
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ if (remain > 0) {
+ out.write(b);
+ --remain;
+ } else {
+ throw new IOException("Writing more bytes than advertised size.");
+ }
+ }
+
+ @Override
+ public void write(byte b[]) throws IOException {
+ write(b, 0, b.length);
+ }
+
+ @Override
+ public void write(byte b[], int off, int len) throws IOException {
+ if (remain >= len) {
+ out.write(b, off, len);
+ remain -= len;
+ } else {
+ throw new IOException("Writing more bytes than advertised size.");
+ }
+ }
+
+ @Override
+ public void flush() throws IOException {
+ out.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed == true) {
+ return;
+ }
+
+ try {
+ if (remain > 0) {
+ throw new IOException("Writing less bytes than advertised size.");
+ }
+ } finally {
+ closed = true;
+ }
+ }
+ }
+}
[50/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
index 156fa3a,0000000..2a1372c
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
@@@ -1,242 -1,0 +1,209 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.admin;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.ClientExec;
+import org.apache.accumulo.core.client.impl.ClientExecReturn;
+import org.apache.accumulo.core.client.impl.MasterClient;
+import org.apache.accumulo.core.client.impl.ServerClient;
+import org.apache.accumulo.core.client.impl.thrift.ClientService;
+import org.apache.accumulo.core.client.impl.thrift.ConfigurationType;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+
+/**
+ * Provides a class for administering the accumulo instance
+ */
+public class InstanceOperationsImpl implements InstanceOperations {
+ private Instance instance;
+ private TCredentials credentials;
+
+ /**
+ * @param instance
+ * the connection information for this instance
+ * @param credentials
+ * the Credential, containing principal and Authentication Token
+ */
+ public InstanceOperationsImpl(Instance instance, TCredentials credentials) {
+ ArgumentChecker.notNull(instance, credentials);
+ this.instance = instance;
+ this.credentials = credentials;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#setProperty(java.lang.String, java.lang.String)
- */
+ @Override
+ public void setProperty(final String property, final String value) throws AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(property, value);
+ MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() {
+ @Override
+ public void execute(MasterClientService.Client client) throws Exception {
+ client.setSystemProperty(Tracer.traceInfo(), credentials, property, value);
+ }
+ });
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#removeProperty(java.lang.String)
- */
+ @Override
+ public void removeProperty(final String property) throws AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(property);
+ MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() {
+ @Override
+ public void execute(MasterClientService.Client client) throws Exception {
+ client.removeSystemProperty(Tracer.traceInfo(), credentials, property);
+ }
+ });
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getSystemConfiguration()
- */
+ @Override
+ public Map<String,String> getSystemConfiguration() throws AccumuloException, AccumuloSecurityException {
+ return ServerClient.execute(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() {
+ @Override
+ public Map<String,String> execute(ClientService.Client client) throws Exception {
+ return client.getConfiguration(Tracer.traceInfo(), credentials, ConfigurationType.CURRENT);
+ }
+ });
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getSiteConfiguration()
- */
+ @Override
+ public Map<String,String> getSiteConfiguration() throws AccumuloException, AccumuloSecurityException {
+ return ServerClient.execute(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() {
+ @Override
+ public Map<String,String> execute(ClientService.Client client) throws Exception {
+ return client.getConfiguration(Tracer.traceInfo(), credentials, ConfigurationType.SITE);
+ }
+ });
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getTabletServers()
- */
-
+ @Override
+ public List<String> getTabletServers() {
+ ZooCache cache = ZooCache.getInstance(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
+ String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
+ List<String> results = new ArrayList<String>();
+ for (String candidate : cache.getChildren(path)) {
+ List<String> children = cache.getChildren(path + "/" + candidate);
+ if (children != null && children.size() > 0) {
+ List<String> copy = new ArrayList<String>(children);
+ Collections.sort(copy);
+ byte[] data = cache.get(path + "/" + candidate + "/" + copy.get(0));
+ if (data != null && !"master".equals(new String(data, Constants.UTF8))) {
+ results.add(candidate);
+ }
+ }
+ }
+ return results;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getActiveScans(java.lang.String)
- */
-
+ @Override
+ public List<ActiveScan> getActiveScans(String tserver) throws AccumuloException, AccumuloSecurityException {
+ Client client = null;
+ try {
+ client = ThriftUtil.getTServerClient(tserver, instance.getConfiguration());
+
+ List<ActiveScan> as = new ArrayList<ActiveScan>();
+ for (org.apache.accumulo.core.tabletserver.thrift.ActiveScan activeScan : client.getActiveScans(Tracer.traceInfo(), credentials)) {
+ try {
+ as.add(new ActiveScan(instance, activeScan));
+ } catch (TableNotFoundException e) {
+ throw new AccumuloException(e);
+ }
+ }
+ return as;
+ } catch (TTransportException e) {
+ throw new AccumuloException(e);
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (TException e) {
+ throw new AccumuloException(e);
+ } finally {
+ if (client != null)
+ ThriftUtil.returnClient(client);
+ }
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#testClassLoad(java.lang.String, java.lang.String)
- */
+ @Override
+ public boolean testClassLoad(final String className, final String asTypeName) throws AccumuloException, AccumuloSecurityException {
+ return ServerClient.execute(instance, new ClientExecReturn<Boolean,ClientService.Client>() {
+ @Override
+ public Boolean execute(ClientService.Client client) throws Exception {
+ return client.checkClass(Tracer.traceInfo(), credentials, className, asTypeName);
+ }
+ });
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getActiveCompactions(java.lang.String)
- */
+ @Override
+ public List<ActiveCompaction> getActiveCompactions(String tserver) throws AccumuloException, AccumuloSecurityException {
+ Client client = null;
+ try {
+ client = ThriftUtil.getTServerClient(tserver, instance.getConfiguration());
+
+ List<ActiveCompaction> as = new ArrayList<ActiveCompaction>();
+ for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction activeCompaction : client.getActiveCompactions(Tracer.traceInfo(), credentials)) {
+ as.add(new ActiveCompaction(instance, activeCompaction));
+ }
+ return as;
+ } catch (TTransportException e) {
+ throw new AccumuloException(e);
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (TException e) {
+ throw new AccumuloException(e);
+ } finally {
+ if (client != null)
+ ThriftUtil.returnClient(client);
+ }
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#ping(java.lang.String)
- */
+ @Override
+ public void ping(String tserver) throws AccumuloException {
+ TTransport transport = null;
+ try {
+ transport = ThriftUtil.createTransport(tserver, instance.getConfiguration().getPort(Property.TSERV_CLIENTPORT), instance.getConfiguration());
+ TabletClientService.Client client = ThriftUtil.createClient(new TabletClientService.Client.Factory(), transport);
+ client.getTabletServerStatus(Tracer.traceInfo(), credentials);
+ } catch (TTransportException e) {
+ throw new AccumuloException(e);
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloException(e);
+ } catch (TException e) {
+ throw new AccumuloException(e);
+ } finally {
+ if (transport != null) {
+ transport.close();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
index 2521c96,0000000..0823656
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
@@@ -1,719 -1,0 +1,687 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.admin;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedSet;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Provides a class for administering tables
+ *
+ */
+
+public interface TableOperations {
+
+ /**
+ * Retrieve a list of tables in Accumulo.
+ *
+ * @return List of tables in accumulo
+ */
+ public SortedSet<String> list();
+
+ /**
+ * A method to check if a table exists in Accumulo.
+ *
+ * @param tableName
+ * the name of the table
+ * @return true if the table exists
+ */
+ public boolean exists(String tableName);
+
+ /**
+ * Create a table with no special configuration
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableExistsException
+ * if the table already exists
+ */
+ public void create(String tableName) throws AccumuloException, AccumuloSecurityException, TableExistsException;
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param limitVersion
+ * Enables/disables the versioning iterator, which will limit the number of Key versions kept.
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableExistsException
+ * if the table already exists
+ */
+ public void create(String tableName, boolean limitVersion) throws AccumuloException, AccumuloSecurityException, TableExistsException;
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param versioningIter
+ * Enables/disables the versioning iterator, which will limit the number of Key versions kept.
+ * @param timeType
+ * specifies logical or real-time based time recording for entries in the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableExistsException
+ * if the table already exists
+ */
+ public void create(String tableName, boolean versioningIter, TimeType timeType) throws AccumuloException, AccumuloSecurityException, TableExistsException;
+
+ /**
+ * Imports a table exported via exportTable and copied via hadoop distcp.
+ *
+ * @param tableName
+ * Name of a table to create and import into.
+ * @param importDir
+ * Directory that contains the files copied by distcp from exportTable
- * @throws TableExistsException
- * @throws AccumuloException
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public void importTable(String tableName, String importDir) throws TableExistsException, AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Exports a table. The tables data is not exported, just table metadata and a list of files to distcp. The table being exported must be offline and stay
+ * offline for the duration of distcp. To avoid losing access to a table it can be cloned and the clone taken offline for export.
+ *
+ * <p>
+ * See docs/examples/README.export
+ *
+ * @param tableName
+ * Name of the table to export.
+ * @param exportDir
+ * An empty directory in HDFS where files containing table metadata and list of files to distcp will be placed.
- * @throws TableNotFoundException
- * @throws AccumuloException
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public void exportTable(String tableName, String exportDir) throws TableNotFoundException, AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Ensures that tablets are split along a set of keys.
+ * <p>
+ * Note that while the documentation for Text specifies that its bytestream should be UTF-8, the encoding is not enforced by operations that work with byte arrays.
+ * <p>
+ * For example, you can create 256 evenly-sliced splits via the following code sample even though the given byte sequences are not valid UTF-8.
+ * <pre>
+ * {@code
+ * TableOperations tableOps = connector.tableOperations();
+ * TreeSet<Text> splits = new TreeSet<Text>();
+ * for (int i = 0; i < 256; i++) {
+ * byte[] bytes = { (byte) i };
+ * splits.add(new Text(bytes));
+ * }
+ * tableOps.addSplits(TABLE_NAME, splits);
+ * }
+ * </pre>
+ *
+ * @param tableName
+ * the name of the table
+ * @param partitionKeys
+ * a sorted set of row key values to pre-split the table on
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ public void addSplits(String tableName, SortedSet<Text> partitionKeys) throws TableNotFoundException, AccumuloException, AccumuloSecurityException;
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @return the split points (end-row names) for the table's current split profile
+ * @throws TableNotFoundException
+ * if the table does not exist
+ * @deprecated since 1.5.0; use {@link #listSplits(String)} instead.
+ */
+ @Deprecated
+ public Collection<Text> getSplits(String tableName) throws TableNotFoundException;
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @return the split points (end-row names) for the table's current split profile
+ * @throws TableNotFoundException
+ * if the table does not exist
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @since 1.5.0
+ */
+ public Collection<Text> listSplits(String tableName) throws TableNotFoundException, AccumuloSecurityException, AccumuloException;
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param maxSplits
+ * specifies the maximum number of splits to return
+ * @return the split points (end-row names) for the table's current split profile, grouped into fewer splits so as not to exceed maxSplits
- * @throws TableNotFoundException
+ * @deprecated since 1.5.0; use {@link #listSplits(String, int)} instead.
+ */
+ @Deprecated
+ public Collection<Text> getSplits(String tableName, int maxSplits) throws TableNotFoundException;
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param maxSplits
+ * specifies the maximum number of splits to return
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @return the split points (end-row names) for the table's current split profile, grouped into fewer splits so as not to exceed maxSplits
- * @throws TableNotFoundException
+ * @since 1.5.0
+ */
+ public Collection<Text> listSplits(String tableName, int maxSplits) throws TableNotFoundException, AccumuloSecurityException, AccumuloException;
+
+ /**
+ * Finds the max row within a given range. To find the max row in a table, pass null for start and end row.
+ *
- * @param tableName
+ * @param auths
+ * find the max row that can seen with these auths
+ * @param startRow
+ * row to start looking at, null means -Infinity
+ * @param startInclusive
+ * determines if the start row is included
+ * @param endRow
+ * row to stop looking at, null means Infinity
+ * @param endInclusive
+ * determines if the end row is included
+ *
+ * @return The max row in the range, or null if there is no visible data in the range.
- *
- * @throws AccumuloSecurityException
- * @throws AccumuloException
- * @throws TableNotFoundException
+ */
+ public Text getMaxRow(String tableName, Authorizations auths, Text startRow, boolean startInclusive, Text endRow, boolean endInclusive)
+ throws TableNotFoundException, AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Merge tablets between (start, end]
+ *
+ * @param tableName
+ * the table to merge
+ * @param start
+ * first tablet to be merged contains the row after this row, null means the first tablet
+ * @param end
+ * last tablet to be merged contains this row, null means the last tablet
+ */
+ public void merge(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
+ /**
+ * Delete rows between (start, end]
+ *
+ * @param tableName
+ * the table to merge
+ * @param start
+ * delete rows after this, null means the first row of the table
+ * @param end
+ * last row to be deleted, inclusive, null means the last row of the table
+ */
+ public void deleteRows(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
+ /**
+ * Starts a full major compaction of the tablets in the range (start, end]. The compaction is preformed even for tablets that have only one file.
+ *
+ * @param tableName
+ * the table to compact
+ * @param start
+ * first tablet to be compacted contains the row after this row, null means the first tablet in table
+ * @param end
+ * last tablet to be merged contains this row, null means the last tablet in table
+ * @param flush
+ * when true, table memory is flushed before compaction starts
+ * @param wait
+ * when true, the call will not return until compactions are finished
+ */
+ public void compact(String tableName, Text start, Text end, boolean flush, boolean wait) throws AccumuloSecurityException, TableNotFoundException,
+ AccumuloException;
+
+ /**
+ * Starts a full major compaction of the tablets in the range (start, end]. The compaction is preformed even for tablets that have only one file.
+ *
+ * @param tableName
+ * the table to compact
+ * @param start
+ * first tablet to be compacted contains the row after this row, null means the first tablet in table
+ * @param end
+ * last tablet to be merged contains this row, null means the last tablet in table
+ * @param iterators
+ * A set of iterators that will be applied to each tablet compacted
+ * @param flush
+ * when true, table memory is flushed before compaction starts
+ * @param wait
+ * when true, the call will not return until compactions are finished
+ * @since 1.5.0
+ */
+ public void compact(String tableName, Text start, Text end, List<IteratorSetting> iterators, boolean flush, boolean wait) throws AccumuloSecurityException,
+ TableNotFoundException, AccumuloException;
+
+ /**
+ * Cancels a user initiated major compaction of a table initiated with {@link #compact(String, Text, Text, boolean, boolean)} or
+ * {@link #compact(String, Text, Text, List, boolean, boolean)}. Compactions of tablets that are currently running may finish, but new compactions of tablets
+ * will not start.
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws TableNotFoundException
+ * if the table does not exist
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @since 1.5.0
+ */
+ public void cancelCompaction(String tableName) throws AccumuloSecurityException, TableNotFoundException, AccumuloException;
+
+ /**
+ * Delete a table
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ public void delete(String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
+ /**
+ * Clone a table from an existing table. The cloned table will have the same data as the source table it was created from. After cloning, the two tables can
+ * mutate independently. Initially the cloned table should not use any extra space, however as the source table and cloned table major compact extra space
+ * will be used by the clone.
+ *
+ * Initially the cloned table is only readable and writable by the user who created it.
+ *
+ * @param srcTableName
+ * the table to clone
+ * @param newTableName
+ * the name of the clone
+ * @param flush
+ * determines if memory is flushed in the source table before cloning.
+ * @param propertiesToSet
+ * the sources tables properties are copied, this allows overriding of those properties
+ * @param propertiesToExclude
+ * do not copy these properties from the source table, just revert to system defaults
+ */
+
+ public void clone(String srcTableName, String newTableName, boolean flush, Map<String,String> propertiesToSet, Set<String> propertiesToExclude)
+ throws AccumuloException, AccumuloSecurityException, TableNotFoundException, TableExistsException;
+
+ /**
+ * Rename a table
+ *
+ * @param oldTableName
+ * the old table name
+ * @param newTableName
+ * the new table name
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the old table name does not exist
+ * @throws TableExistsException
+ * if the new table name already exists
+ */
+ public void rename(String oldTableName, String newTableName) throws AccumuloSecurityException, TableNotFoundException, AccumuloException,
+ TableExistsException;
+
+ /**
+ * Initiate a flush of a table's data that is in memory
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ *
+ * @deprecated As of release 1.4, replaced by {@link #flush(String, Text, Text, boolean)}
+ */
+ @Deprecated
+ public void flush(String tableName) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Flush a table's data that is currently in memory.
+ *
+ * @param tableName
+ * the name of the table
+ * @param wait
+ * if true the call will not return until all data present in memory when the call was is flushed if false will initiate a flush of data in memory,
+ * but will not wait for it to complete
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
- * @throws TableNotFoundException
+ */
+ public void flush(String tableName, Text start, Text end, boolean wait) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
+ /**
+ * Sets a property on a table. Note that it may take a short period of time (a second) to propagate the change everywhere.
+ *
+ * @param tableName
+ * the name of the table
+ * @param property
+ * the name of a per-table property
+ * @param value
+ * the value to set a per-table property to
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ public void setProperty(String tableName, String property, String value) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Removes a property from a table. Note that it may take a short period of time (a second) to propagate the change everywhere.
+ *
+ * @param tableName
+ * the name of the table
+ * @param property
+ * the name of a per-table property
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ public void removeProperty(String tableName, String property) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Gets properties of a table. Note that recently changed properties may not be available immediately.
+ *
+ * @param tableName
+ * the name of the table
+ * @return all properties visible by this table (system and per-table properties). Note that recently changed
+ * properties may not be visible immediately.
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ public Iterable<Entry<String,String>> getProperties(String tableName) throws AccumuloException, TableNotFoundException;
+
+ /**
+ * Sets a table's locality groups. A table's locality groups can be changed at any time.
+ *
+ * @param tableName
+ * the name of the table
+ * @param groups
+ * mapping of locality group names to column families in the locality group
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ public void setLocalityGroups(String tableName, Map<String,Set<Text>> groups) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
+ /**
+ *
+ * Gets the locality groups currently set for a table.
+ *
+ * @param tableName
+ * the name of the table
+ * @return mapping of locality group names to column families in the locality group
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ public Map<String,Set<Text>> getLocalityGroups(String tableName) throws AccumuloException, TableNotFoundException;
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param range
+ * a range to split
+ * @param maxSplits
+ * the maximum number of splits
+ * @return the range, split into smaller ranges that fall on boundaries of the table's split points as evenly as possible
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ public Set<Range> splitRangeByTablets(String tableName, Range range, int maxSplits) throws AccumuloException, AccumuloSecurityException,
+ TableNotFoundException;
+
+ /**
+ * Bulk import all the files in a directory into a table.
+ *
+ * @param tableName
+ * the name of the table
+ * @param dir
+ * the HDFS directory to find files for importing
+ * @param failureDir
+ * the HDFS directory to place files that failed to be imported, must exist and be empty
+ * @param setTime
+ * override the time values in the input files, and use the current time for all mutations
+ * @throws IOException
+ * when there is an error reading/writing to HDFS
+ * @throws AccumuloException
+ * when there is a general accumulo error
+ * @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
+ * @throws TableNotFoundException
+ * when the table no longer exists
+ *
+ */
+ public void importDirectory(String tableName, String dir, String failureDir, boolean setTime) throws TableNotFoundException, IOException, AccumuloException,
+ AccumuloSecurityException;
+
+ /**
+ *
+ * @param tableName
+ * the table to take offline
+ * @throws AccumuloException
+ * when there is a general accumulo error
+ * @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
- * @throws TableNotFoundException
+ */
+ public void offline(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
+
+ /**
+ *
+ * @param tableName
+ * the table to take online
+ * @throws AccumuloException
+ * when there is a general accumulo error
+ * @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
- * @throws TableNotFoundException
+ */
+ public void online(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
+
+ /**
+ * Clears the tablet locator cache for a specified table
+ *
+ * @param tableName
+ * the name of the table
+ * @throws TableNotFoundException
+ * if table does not exist
+ */
+ public void clearLocatorCache(String tableName) throws TableNotFoundException;
+
+ /**
+ * Get a mapping of table name to internal table id.
+ *
+ * @return the map from table name to internal table id
+ */
+ public Map<String,String> tableIdMap();
+
+ /**
+ * Add an iterator to a table on all scopes.
+ *
+ * @param tableName
+ * the name of the table
+ * @param setting
+ * object specifying the properties of the iterator
+ * @throws AccumuloSecurityException
+ * thrown if the user does not have the ability to set properties on the table
- * @throws AccumuloException
+ * @throws TableNotFoundException
+ * throw if the table no longer exists
+ * @throws IllegalArgumentException
+ * if the setting conflicts with any existing iterators
+ */
+ public void attachIterator(String tableName, IteratorSetting setting) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
+
+ /**
+ * Add an iterator to a table on the given scopes.
+ *
+ * @param tableName
+ * the name of the table
+ * @param setting
+ * object specifying the properties of the iterator
+ * @throws AccumuloSecurityException
+ * thrown if the user does not have the ability to set properties on the table
- * @throws AccumuloException
+ * @throws TableNotFoundException
+ * throw if the table no longer exists
+ * @throws IllegalArgumentException
+ * if the setting conflicts with any existing iterators
+ */
+ public void attachIterator(String tableName, IteratorSetting setting, EnumSet<IteratorScope> scopes) throws AccumuloSecurityException, AccumuloException,
+ TableNotFoundException;
+
+ /**
+ * Remove an iterator from a table by name.
+ *
+ * @param tableName
+ * the name of the table
+ * @param name
+ * the name of the iterator
+ * @param scopes
+ * the scopes of the iterator
+ * @throws AccumuloSecurityException
+ * thrown if the user does not have the ability to set properties on the table
- * @throws AccumuloException
+ * @throws TableNotFoundException
+ * throw if the table no longer exists
+ */
+ public void removeIterator(String tableName, String name, EnumSet<IteratorScope> scopes) throws AccumuloSecurityException, AccumuloException,
+ TableNotFoundException;
+
+ /**
+ * Get the settings for an iterator.
+ *
+ * @param tableName
+ * the name of the table
+ * @param name
+ * the name of the iterator
+ * @param scope
+ * the scope of the iterator
+ * @return the settings for this iterator
+ * @throws AccumuloSecurityException
+ * thrown if the user does not have the ability to set properties on the table
- * @throws AccumuloException
+ * @throws TableNotFoundException
+ * throw if the table no longer exists
+ */
+ public IteratorSetting getIteratorSetting(String tableName, String name, IteratorScope scope) throws AccumuloSecurityException, AccumuloException,
+ TableNotFoundException;
+
+ /**
+ * Get a list of iterators for this table.
+ *
+ * @param tableName
+ * the name of the table
+ * @return a set of iterator names
- * @throws AccumuloSecurityException
- * @throws AccumuloException
- * @throws TableNotFoundException
+ */
+ public Map<String,EnumSet<IteratorScope>> listIterators(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
+
+ /**
+ * Check whether a given iterator configuration conflicts with existing configuration; in particular, determine if the name or priority are already in use for
+ * the specified scopes.
+ *
+ * @param tableName
+ * the name of the table
+ * @param setting
+ * object specifying the properties of the iterator
- * @throws AccumuloException
- * @throws TableNotFoundException
+ * @throws IllegalStateException
+ * if the setting conflicts with any existing iterators
+ */
+ public void checkIteratorConflicts(String tableName, IteratorSetting setting, EnumSet<IteratorScope> scopes) throws AccumuloException, TableNotFoundException;
+
+ /**
+ * Add a new constraint to a table.
+ *
+ * @param tableName
+ * the name of the table
+ * @param constraintClassName
+ * the full name of the constraint class
+ * @return the unique number assigned to the constraint
+ * @throws AccumuloException
+ * thrown if the constraint has already been added to the table or if there are errors in the configuration of existing constraints
+ * @throws AccumuloSecurityException
+ * thrown if the user doesn't have permission to add the constraint
- * @throws TableNotFoundException
+ * @since 1.5.0
+ */
+ public int addConstraint(String tableName, String constraintClassName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
+ /**
+ * Remove a constraint from a table.
+ *
+ * @param tableName
+ * the name of the table
+ * @param number
+ * the unique number assigned to the constraint
- * @throws AccumuloException
+ * @throws AccumuloSecurityException
+ * thrown if the user doesn't have permission to remove the constraint
+ * @since 1.5.0
+ */
+ public void removeConstraint(String tableName, int number) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * List constraints on a table with their assigned numbers.
+ *
+ * @param tableName
+ * the name of the table
+ * @return a map from constraint class name to assigned number
+ * @throws AccumuloException
+ * thrown if there are errors in the configuration of existing constraints
- * @throws TableNotFoundException
+ * @since 1.5.0
+ */
+ public Map<String,Integer> listConstraints(String tableName) throws AccumuloException, TableNotFoundException;
+
+ /**
+ * Test to see if the instance can load the given class as the given type. This check uses the table classpath if it is set.
+ *
- * @param className
- * @param asTypeName
+ * @return true if the instance can load the given class as the given type, false otherwise
- * @throws AccumuloException
- *
+ *
+ * @since 1.5.0
+ */
+ public boolean testClassLoad(String tableName, final String className, final String asTypeName) throws AccumuloException, AccumuloSecurityException,
+ TableNotFoundException;
+}
[25/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
index 8835b1c,0000000..53ea0e8
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
@@@ -1,676 -1,0 +1,672 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators.user;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.NoSuchElementException;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.OptionDescriber;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.WrappingIterator;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.VisibilityEvaluator;
+import org.apache.accumulo.core.security.VisibilityParseException;
+import org.apache.accumulo.core.util.BadArgumentException;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.commons.collections.BufferOverflowException;
+import org.apache.commons.collections.map.LRUMap;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+/**
+ * The TransformingIterator allows portions of a key (except for the row) to be transformed. This iterator handles the details that come with modifying keys
+ * (i.e., that the sort order could change). In order to do so, however, the iterator must put all keys sharing the same prefix in memory. Prefix is defined as
+ * the parts of the key that are not modified by this iterator. That is, if the iterator modifies column qualifier and timestamp, then the prefix is row and
+ * column family. In that case, the iterator must load all column qualifiers for each row/column family pair into memory. Given this constraint, care must be
+ * taken by users of this iterator to ensure it is not run in such a way that will overrun memory in a tablet server.
+ * <p>
+ * If the implementing iterator is transforming column families, then it must also override {@code untransformColumnFamilies(Collection)} to handle the case
+ * when column families are fetched at scan time. The fetched column families will/must be in the transformed space, and the untransformed column families need
+ * to be passed to this iterator's source. If it is not possible to write a reverse transformation (e.g., the column family transformation depends on the row
+ * value or something like that), then the iterator must not fetch specific column families (or only fetch column families that are known to not transform at
+ * all).
+ * <p>
+ * If the implementing iterator is transforming column visibilities, then users must be careful NOT to fetch column qualifiers from the scanner. The reason for
+ * this is due to ACCUMULO-??? (insert issue number).
+ * <p>
+ * If the implementing iterator is transforming column visibilities, then the user should be sure to supply authorizations via the {@link #AUTH_OPT} iterator
+ * option (note that this is only necessary for scan scope iterators). The supplied authorizations should be in the transformed space, but the authorizations
+ * supplied to the scanner should be in the untransformed space. That is, if the iterator transforms A to 1, B to 2, C to 3, etc, then the auths supplied when
+ * the scanner is constructed should be A,B,C,... and the auths supplied to the iterator should be 1,2,3,... The reason for this is that the scanner performs
+ * security filtering before this iterator is called, so the authorizations need to be in the original untransformed space. Since the iterator can transform
+ * visibilities, it is possible that it could produce visibilities that the user cannot see, so the transformed keys must be tested to ensure the user is
+ * allowed to view them. Note that this test is not necessary when the iterator is not used in the scan scope since no security filtering is performed during
+ * major and minor compactions. It should also be noted that this iterator implements the security filtering rather than relying on a follow-on iterator to do
+ * it so that we ensure the test is performed.
+ */
+abstract public class TransformingIterator extends WrappingIterator implements OptionDescriber {
+ public static final String AUTH_OPT = "authorizations";
+ public static final String MAX_BUFFER_SIZE_OPT = "maxBufferSize";
+ private static final long DEFAULT_MAX_BUFFER_SIZE = 10000000;
-
++
+ protected Logger log = Logger.getLogger(getClass());
-
++
+ protected ArrayList<Pair<Key,Value>> keys = new ArrayList<Pair<Key,Value>>();
+ protected int keyPos = -1;
+ protected boolean scanning;
+ protected Range seekRange;
+ protected Collection<ByteSequence> seekColumnFamilies;
+ protected boolean seekColumnFamiliesInclusive;
-
++
+ private VisibilityEvaluator ve = null;
+ private LRUMap visibleCache = null;
+ private LRUMap parsedVisibilitiesCache = null;
+ private long maxBufferSize;
-
++
+ private static Comparator<Pair<Key,Value>> keyComparator = new Comparator<Pair<Key,Value>>() {
+ @Override
+ public int compare(Pair<Key,Value> o1, Pair<Key,Value> o2) {
+ return o1.getFirst().compareTo(o2.getFirst());
+ }
+ };
-
++
+ public TransformingIterator() {}
-
++
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ super.init(source, options, env);
+ scanning = IteratorScope.scan.equals(env.getIteratorScope());
+ if (scanning) {
+ String auths = options.get(AUTH_OPT);
+ if (auths != null && !auths.isEmpty()) {
+ ve = new VisibilityEvaluator(new Authorizations(auths.getBytes(Constants.UTF8)));
+ visibleCache = new LRUMap(100);
+ }
+ }
-
++
+ if (options.containsKey(MAX_BUFFER_SIZE_OPT)) {
+ maxBufferSize = AccumuloConfiguration.getMemoryInBytes(options.get(MAX_BUFFER_SIZE_OPT));
+ } else {
+ maxBufferSize = DEFAULT_MAX_BUFFER_SIZE;
+ }
-
++
+ parsedVisibilitiesCache = new LRUMap(100);
+ }
-
++
+ @Override
+ public IteratorOptions describeOptions() {
+ String desc = "This iterator allows ranges of key to be transformed (with the exception of row transformations).";
+ String authDesc = "Comma-separated list of user's scan authorizations. "
+ + "If excluded or empty, then no visibility check is performed on transformed keys.";
+ String bufferDesc = "Maximum buffer size (in accumulo memory spec) to use for buffering keys before throwing a BufferOverflowException. "
+ + "Users should keep this limit in mind when deciding what to transform. That is, if transforming the column family for example, then all "
+ + "keys sharing the same row and column family must fit within this limit (along with their associated values)";
+ HashMap<String,String> namedOptions = new HashMap<String,String>();
+ namedOptions.put(AUTH_OPT, authDesc);
+ namedOptions.put(MAX_BUFFER_SIZE_OPT, bufferDesc);
+ return new IteratorOptions(getClass().getSimpleName(), desc, namedOptions, null);
+ }
-
++
+ @Override
+ public boolean validateOptions(Map<String,String> options) {
-
++
+ for (Entry<String,String> option : options.entrySet()) {
+ try {
+ if (option.getKey().equals(AUTH_OPT)) {
+ new Authorizations(option.getValue().getBytes(Constants.UTF8));
+ } else if (option.getKey().equals(MAX_BUFFER_SIZE_OPT)) {
+ AccumuloConfiguration.getMemoryInBytes(option.getValue());
+ }
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Failed to parse opt " + option.getKey() + " " + option.getValue(), e);
+ }
+ }
-
++
+ return true;
+ }
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ TransformingIterator copy;
-
++
+ try {
+ copy = getClass().newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
-
++
+ copy.setSource(getSource().deepCopy(env));
-
++
+ copy.scanning = scanning;
+ copy.keyPos = keyPos;
+ copy.keys.addAll(keys);
+ copy.seekRange = (seekRange == null) ? null : new Range(seekRange);
+ copy.seekColumnFamilies = (seekColumnFamilies == null) ? null : new HashSet<ByteSequence>(seekColumnFamilies);
+ copy.seekColumnFamiliesInclusive = seekColumnFamiliesInclusive;
-
++
+ copy.ve = ve;
+ if (visibleCache != null) {
+ copy.visibleCache = new LRUMap(visibleCache.maxSize());
+ copy.visibleCache.putAll(visibleCache);
+ }
-
++
+ if (parsedVisibilitiesCache != null) {
+ copy.parsedVisibilitiesCache = new LRUMap(parsedVisibilitiesCache.maxSize());
+ copy.parsedVisibilitiesCache.putAll(parsedVisibilitiesCache);
+ }
-
++
+ copy.maxBufferSize = maxBufferSize;
-
++
+ return copy;
+ }
-
++
+ @Override
+ public boolean hasTop() {
+ return keyPos >= 0 && keyPos < keys.size();
+ }
-
++
+ @Override
+ public Key getTopKey() {
+ return hasTop() ? keys.get(keyPos).getFirst() : null;
+ }
-
++
+ @Override
+ public Value getTopValue() {
+ return hasTop() ? keys.get(keyPos).getSecond() : null;
+ }
-
++
+ @Override
+ public void next() throws IOException {
+ // Move on to the next entry since we returned the entry at keyPos before
+ if (keyPos >= 0)
+ keyPos++;
-
++
+ // If we emptied out the transformed key map then transform the next key
+ // set from the source. It’s possible that transformation could produce keys
+ // that are outside of our range or are not visible to the end user, so after the
+ // call below we might not have added any keys to the map. Keep going until
+ // we either get some keys in the map or exhaust the source iterator.
+ while (!hasTop() && super.hasTop())
+ transformKeys();
+ }
-
++
+ @Override
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
+ seekRange = new Range(range);
+ seekColumnFamilies = columnFamilies;
+ seekColumnFamiliesInclusive = inclusive;
-
++
+ // Seek the source iterator, but use a recalculated range that ensures
+ // we see all keys with the same "prefix." We need to do this since
+ // transforming could change the sort order and transformed keys that
+ // are before the range start could be inside the range after transformation.
+ super.seek(computeReseekRange(range), untransformColumnFamilies(columnFamilies), inclusive);
-
++
+ // Range clipping could cause us to trim out all the keys we transformed.
+ // Keep looping until we either have some keys in the output range, or have
+ // exhausted the source iterator.
+ keyPos = -1; // “Clear” list so hasTop returns false to get us into the loop (transformKeys actually clears)
+ while (!hasTop() && super.hasTop()) {
+ // Build up a sorted list of all keys for the same prefix. When
+ // people ask for keys, return from this list first until it is empty
+ // before incrementing the source iterator.
+ transformKeys();
+ }
+ }
-
++
+ private static class RangeIterator implements SortedKeyValueIterator<Key,Value> {
-
++
+ private SortedKeyValueIterator<Key,Value> source;
+ private Key prefixKey;
+ private PartialKey keyPrefix;
+ private boolean hasTop = false;
-
++
+ RangeIterator(SortedKeyValueIterator<Key,Value> source, Key prefixKey, PartialKey keyPrefix) {
+ this.source = source;
+ this.prefixKey = prefixKey;
+ this.keyPrefix = keyPrefix;
+ }
-
++
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ throw new UnsupportedOperationException();
+ }
-
++
+ @Override
+ public boolean hasTop() {
+ // only have a top if the prefix matches
+ return hasTop = source.hasTop() && source.getTopKey().equals(prefixKey, keyPrefix);
+ }
-
++
+ @Override
+ public void next() throws IOException {
+ // do not let user advance too far and try to avoid reexecuting hasTop()
+ if (!hasTop && !hasTop())
+ throw new NoSuchElementException();
+ hasTop = false;
+ source.next();
+ }
-
++
+ @Override
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
+ throw new UnsupportedOperationException();
+ }
-
++
+ @Override
+ public Key getTopKey() {
+ return source.getTopKey();
+ }
-
++
+ @Override
+ public Value getTopValue() {
+ return source.getTopValue();
+ }
-
++
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ throw new UnsupportedOperationException();
+ }
-
++
+ }
-
++
+ /**
+ * Reads all keys matching the first key's prefix from the source iterator, transforms them, and sorts the resulting keys. Transformed keys that fall outside
+ * of our seek range or can't be seen by the user are excluded.
+ */
+ protected void transformKeys() throws IOException {
+ keyPos = -1;
+ keys.clear();
+ final Key prefixKey = super.hasTop() ? new Key(super.getTopKey()) : null;
-
++
+ transformRange(new RangeIterator(getSource(), prefixKey, getKeyPrefix()), new KVBuffer() {
-
++
+ long appened = 0;
-
++
+ @Override
+ public void append(Key key, Value val) {
+ // ensure the key provided by the user has the correct prefix
+ if (!key.equals(prefixKey, getKeyPrefix()))
+ throw new IllegalArgumentException("Key prefixes are not equal " + key + " " + prefixKey);
-
++
+ // Transformation could have produced a key that falls outside
+ // of the seek range, or one that the user cannot see. Check
+ // these before adding it to the output list.
+ if (includeTransformedKey(key)) {
-
++
+ // try to defend against a scan or compaction using all memory in a tablet server
+ if (appened > maxBufferSize)
+ throw new BufferOverflowException("Exceeded buffer size of " + maxBufferSize + ", prefixKey: " + prefixKey);
-
++
+ if (getSource().hasTop() && key == getSource().getTopKey())
+ key = new Key(key);
+ keys.add(new Pair<Key,Value>(key, new Value(val)));
+ appened += (key.getSize() + val.getSize() + 128);
+ }
+ }
+ });
-
++
+ // consume any key in range that user did not consume
+ while (super.hasTop() && super.getTopKey().equals(prefixKey, getKeyPrefix())) {
+ super.next();
+ }
-
++
+ if (!keys.isEmpty()) {
+ Collections.sort(keys, keyComparator);
+ keyPos = 0;
+ }
+ }
-
++
+ /**
+ * Determines whether or not to include {@code transformedKey} in the output. It is possible that transformation could have produced a key that falls outside
+ * of the seek range, a key with a visibility the user can't see, a key with a visibility that doesn't parse, or a key with a column family that wasn't
+ * fetched. We only do some checks (outside the range, user can see) if we're scanning. The range check is not done for major/minor compaction since seek
+ * ranges won't be in our transformed key space and we will never change the row so we can't produce keys that would fall outside the tablet anyway.
- *
++ *
+ * @param transformedKey
+ * the key to check
+ * @return {@code true} if the key should be included and {@code false} if not
+ */
+ protected boolean includeTransformedKey(Key transformedKey) {
+ boolean include = canSee(transformedKey);
+ if (scanning && seekRange != null) {
+ include = include && seekRange.contains(transformedKey);
+ }
+ return include;
+ }
-
++
+ /**
+ * Indicates whether or not the user is able to see {@code key}. If the user has not supplied authorizations, or the iterator is not in the scan scope, then
+ * this method simply returns {@code true}. Otherwise, {@code key}'s column visibility is tested against the user-supplied authorizations, and the test result
+ * is returned. For performance, the test results are cached so that the same visibility is not tested multiple times.
- *
++ *
+ * @param key
+ * the key to test
+ * @return {@code true} if the key is visible or iterator is not scanning, and {@code false} if not
+ */
+ protected boolean canSee(Key key) {
+ // Ensure that the visibility (which could have been transformed) parses. Must always do this check, even if visibility is not evaluated.
+ ByteSequence visibility = key.getColumnVisibilityData();
+ ColumnVisibility colVis = null;
+ Boolean parsed = (Boolean) parsedVisibilitiesCache.get(visibility);
+ if (parsed == null) {
+ try {
+ colVis = new ColumnVisibility(visibility.toArray());
+ parsedVisibilitiesCache.put(visibility, Boolean.TRUE);
+ } catch (BadArgumentException e) {
+ log.error("Parse error after transformation : " + visibility);
+ parsedVisibilitiesCache.put(visibility, Boolean.FALSE);
+ if (scanning) {
+ return false;
+ } else {
+ throw e;
+ }
+ }
+ } else if (!parsed) {
+ if (scanning)
+ return false;
+ else
+ throw new IllegalStateException();
+ }
-
++
+ Boolean visible = canSeeColumnFamily(key);
-
++
+ if (!scanning || !visible || ve == null || visibleCache == null || visibility.length() == 0)
+ return visible;
-
++
+ visible = (Boolean) visibleCache.get(visibility);
+ if (visible == null) {
+ try {
+ if (colVis == null)
+ colVis = new ColumnVisibility(visibility.toArray());
+ visible = ve.evaluate(colVis);
+ visibleCache.put(visibility, visible);
+ } catch (VisibilityParseException e) {
+ log.error("Parse Error", e);
+ visible = Boolean.FALSE;
+ } catch (BadArgumentException e) {
+ log.error("Parse Error", e);
+ visible = Boolean.FALSE;
+ }
+ }
-
++
+ return visible;
+ }
-
++
+ /**
+ * Indicates whether or not {@code key} can be seen, according to the fetched column families for this iterator.
- *
++ *
+ * @param key
+ * the key whose column family is to be tested
+ * @return {@code true} if {@code key}'s column family is one of those fetched in the set passed to our {@link #seek(Range, Collection, boolean)} method
+ */
+ protected boolean canSeeColumnFamily(Key key) {
+ boolean visible = true;
+ if (seekColumnFamilies != null) {
+ ByteSequence columnFamily = key.getColumnFamilyData();
+ if (seekColumnFamiliesInclusive)
+ visible = seekColumnFamilies.contains(columnFamily);
+ else
+ visible = !seekColumnFamilies.contains(columnFamily);
+ }
+ return visible;
+ }
-
++
+ /**
+ * Possibly expand {@code range} to include everything for the key prefix we are working with. That is, if our prefix is ROW_COLFAM, then we need to expand
+ * the range so we're sure to include all entries having the same row and column family as the start/end of the range.
- *
++ *
+ * @param range
+ * the range to expand
+ * @return the modified range
+ */
+ protected Range computeReseekRange(Range range) {
+ Key startKey = range.getStartKey();
+ boolean startKeyInclusive = range.isStartKeyInclusive();
+ // If anything after the prefix is set, then clip the key so we include
+ // everything for the prefix.
+ if (isSetAfterPart(startKey, getKeyPrefix())) {
+ startKey = copyPartialKey(startKey, getKeyPrefix());
+ startKeyInclusive = true;
+ }
+ Key endKey = range.getEndKey();
+ boolean endKeyInclusive = range.isEndKeyInclusive();
+ if (isSetAfterPart(endKey, getKeyPrefix())) {
+ endKey = endKey.followingKey(getKeyPrefix());
+ endKeyInclusive = true;
+ }
+ return new Range(startKey, startKeyInclusive, endKey, endKeyInclusive);
+ }
-
++
+ /**
+ * Indicates whether or not any part of {@code key} excluding {@code part} is set. For example, if part is ROW_COLFAM_COLQUAL, then this method determines
+ * whether or not the column visibility, timestamp, or delete flag is set on {@code key}.
- *
++ *
+ * @param key
+ * the key to check
+ * @param part
+ * the part of the key that doesn't need to be checked (everything after does)
+ * @return {@code true} if anything after {@code part} is set on {@code key}, and {@code false} if not
+ */
+ protected boolean isSetAfterPart(Key key, PartialKey part) {
+ boolean isSet = false;
+ if (key != null) {
+ // Breaks excluded on purpose.
+ switch (part) {
+ case ROW:
+ isSet = isSet || key.getColumnFamilyData().length() > 0;
+ case ROW_COLFAM:
+ isSet = isSet || key.getColumnQualifierData().length() > 0;
+ case ROW_COLFAM_COLQUAL:
+ isSet = isSet || key.getColumnVisibilityData().length() > 0;
+ case ROW_COLFAM_COLQUAL_COLVIS:
+ isSet = isSet || key.getTimestamp() < Long.MAX_VALUE;
+ case ROW_COLFAM_COLQUAL_COLVIS_TIME:
+ isSet = isSet || key.isDeleted();
+ case ROW_COLFAM_COLQUAL_COLVIS_TIME_DEL:
+ break;
+ }
+ }
+ return isSet;
+ }
-
++
+ /**
+ * Creates a copy of {@code key}, copying only the parts of the key specified in {@code part}. For example, if {@code part} is ROW_COLFAM_COLQUAL, then this
+ * method would copy the row, column family, and column qualifier from {@code key} into a new key.
- *
++ *
+ * @param key
+ * the key to copy
+ * @param part
+ * the parts of {@code key} to copy
+ * @return the new key containing {@code part} of {@code key}
+ */
+ protected Key copyPartialKey(Key key, PartialKey part) {
+ Key keyCopy;
+ switch (part) {
+ case ROW:
+ keyCopy = new Key(key.getRow());
+ break;
+ case ROW_COLFAM:
+ keyCopy = new Key(key.getRow(), key.getColumnFamily());
+ break;
+ case ROW_COLFAM_COLQUAL:
+ keyCopy = new Key(key.getRow(), key.getColumnFamily(), key.getColumnQualifier());
+ break;
+ case ROW_COLFAM_COLQUAL_COLVIS:
+ keyCopy = new Key(key.getRow(), key.getColumnFamily(), key.getColumnQualifier(), key.getColumnVisibility());
+ break;
+ case ROW_COLFAM_COLQUAL_COLVIS_TIME:
+ keyCopy = new Key(key.getRow(), key.getColumnFamily(), key.getColumnQualifier(), key.getColumnVisibility(), key.getTimestamp());
+ break;
+ default:
+ throw new IllegalArgumentException("Unsupported key part: " + part);
+ }
+ return keyCopy;
+ }
-
++
+ /**
+ * Make a new key with all parts (including delete flag) coming from {@code originalKey} but use {@code newColFam} as the column family.
+ */
+ protected Key replaceColumnFamily(Key originalKey, Text newColFam) {
+ byte[] row = originalKey.getRowData().toArray();
+ byte[] cf = newColFam.getBytes();
+ byte[] cq = originalKey.getColumnQualifierData().toArray();
+ byte[] cv = originalKey.getColumnVisibilityData().toArray();
+ long timestamp = originalKey.getTimestamp();
+ Key newKey = new Key(row, 0, row.length, cf, 0, newColFam.getLength(), cq, 0, cq.length, cv, 0, cv.length, timestamp);
+ newKey.setDeleted(originalKey.isDeleted());
+ return newKey;
+ }
-
++
+ /**
+ * Make a new key with all parts (including delete flag) coming from {@code originalKey} but use {@code newColQual} as the column qualifier.
+ */
+ protected Key replaceColumnQualifier(Key originalKey, Text newColQual) {
+ byte[] row = originalKey.getRowData().toArray();
+ byte[] cf = originalKey.getColumnFamilyData().toArray();
+ byte[] cq = newColQual.getBytes();
+ byte[] cv = originalKey.getColumnVisibilityData().toArray();
+ long timestamp = originalKey.getTimestamp();
+ Key newKey = new Key(row, 0, row.length, cf, 0, cf.length, cq, 0, newColQual.getLength(), cv, 0, cv.length, timestamp);
+ newKey.setDeleted(originalKey.isDeleted());
+ return newKey;
+ }
-
++
+ /**
+ * Make a new key with all parts (including delete flag) coming from {@code originalKey} but use {@code newColVis} as the column visibility.
+ */
+ protected Key replaceColumnVisibility(Key originalKey, Text newColVis) {
+ byte[] row = originalKey.getRowData().toArray();
+ byte[] cf = originalKey.getColumnFamilyData().toArray();
+ byte[] cq = originalKey.getColumnQualifierData().toArray();
+ byte[] cv = newColVis.getBytes();
+ long timestamp = originalKey.getTimestamp();
+ Key newKey = new Key(row, 0, row.length, cf, 0, cf.length, cq, 0, cq.length, cv, 0, newColVis.getLength(), timestamp);
+ newKey.setDeleted(originalKey.isDeleted());
+ return newKey;
+ }
-
++
+ /**
+ * Make a new key with a column family, column qualifier, and column visibility. Copy the rest of the parts of the key (including delete flag) from
+ * {@code originalKey}.
+ */
+ protected Key replaceKeyParts(Key originalKey, Text newColFam, Text newColQual, Text newColVis) {
+ byte[] row = originalKey.getRowData().toArray();
+ byte[] cf = newColFam.getBytes();
+ byte[] cq = newColQual.getBytes();
+ byte[] cv = newColVis.getBytes();
+ long timestamp = originalKey.getTimestamp();
+ Key newKey = new Key(row, 0, row.length, cf, 0, newColFam.getLength(), cq, 0, newColQual.getLength(), cv, 0, newColVis.getLength(), timestamp);
+ newKey.setDeleted(originalKey.isDeleted());
+ return newKey;
+ }
-
++
+ /**
+ * Make a new key with a column qualifier, and column visibility. Copy the rest of the parts of the key (including delete flag) from {@code originalKey}.
+ */
+ protected Key replaceKeyParts(Key originalKey, Text newColQual, Text newColVis) {
+ byte[] row = originalKey.getRowData().toArray();
+ byte[] cf = originalKey.getColumnFamilyData().toArray();
+ byte[] cq = newColQual.getBytes();
+ byte[] cv = newColVis.getBytes();
+ long timestamp = originalKey.getTimestamp();
+ Key newKey = new Key(row, 0, row.length, cf, 0, cf.length, cq, 0, newColQual.getLength(), cv, 0, newColVis.getLength(), timestamp);
+ newKey.setDeleted(originalKey.isDeleted());
+ return newKey;
+ }
-
++
+ /**
+ * Reverses the transformation applied to column families that are fetched at seek time. If this iterator is transforming column families, then this method
+ * should be overridden to reverse the transformation on the supplied collection of column families. This is necessary since the fetch/seek will be performed
+ * in the transformed space, but when passing the column family set on to the source, the column families need to be in the untransformed space.
- *
++ *
+ * @param columnFamilies
+ * the column families that have been fetched at seek time
+ * @return the untransformed column families that would transform info {@code columnFamilies}
+ */
+ protected Collection<ByteSequence> untransformColumnFamilies(Collection<ByteSequence> columnFamilies) {
+ return columnFamilies;
+ }
-
++
+ /**
+ * Indicates the prefix of keys that will be transformed by this iterator. In other words, this is the part of the key that will <i>not</i> be transformed by
+ * this iterator. For example, if this method returns ROW_COLFAM, then {@link #transformKeys()} may be changing the column qualifier, column visibility, or
+ * timestamp, but it won't be changing the row or column family.
- *
++ *
+ * @return the part of the key this iterator is not transforming
+ */
+ abstract protected PartialKey getKeyPrefix();
-
++
+ public static interface KVBuffer {
+ void append(Key key, Value val);
+ }
-
++
+ /**
+ * Transforms {@code input}. This method must not change the row part of the key, and must only change the parts of the key after the return value of
+ * {@link #getKeyPrefix()}. Implementors must also remember to copy the delete flag from {@code originalKey} onto the new key. Or, implementors should use one
+ * of the helper methods to produce the new key. See any of the replaceKeyParts methods.
- *
++ *
+ * @param input
+ * An iterator over a group of keys with the same prefix. This iterator provides an efficient view, bounded by the prefix, of the underlying iterator
+ * and can not be seeked.
+ * @param output
+ * An output buffer that holds transformed key values. All key values added to the buffer must have the same prefix as the input keys.
- * @throws IOException
+ * @see #replaceColumnFamily(Key, Text)
+ * @see #replaceColumnQualifier(Key, Text)
+ * @see #replaceColumnVisibility(Key, Text)
+ * @see #replaceKeyParts(Key, Text, Text)
+ * @see #replaceKeyParts(Key, Text, Text, Text)
+ */
+ abstract protected void transformRange(SortedKeyValueIterator<Key,Value> input, KVBuffer output) throws IOException;
-
++
+ /**
- * Configure authoriations used for post transformation filtering.
- *
- * @param config
- * @param auths
++ * Configure authorizations used for post transformation filtering.
++ *
+ */
+ public static void setAuthorizations(IteratorSetting config, Authorizations auths) {
+ config.addOption(AUTH_OPT, auths.serialize());
+ }
-
++
+ /**
+ * Configure the maximum amount of memory that can be used for transformation. If this memory is exceeded an exception will be thrown.
- *
- * @param config
++ *
+ * @param maxBufferSize
+ * size in bytes
+ */
+ public static void setMaxBufferSize(IteratorSetting config, long maxBufferSize) {
+ config.addOption(MAX_BUFFER_SIZE_OPT, maxBufferSize + "");
+ }
-
++
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/user/VersioningIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/VersioningIterator.java
index 7804aa4,0000000..2fc3a27
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/VersioningIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/VersioningIterator.java
@@@ -1,172 -1,0 +1,169 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators.user;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.core.iterators.OptionDescriber;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.WrappingIterator;
+
+public class VersioningIterator extends WrappingIterator implements OptionDescriber {
+ private final int maxCount = 10;
+
+ private Key currentKey = new Key();
+ private int numVersions;
+ protected int maxVersions;
+
+ private Range range;
+ private Collection<ByteSequence> columnFamilies;
+ private boolean inclusive;
+
+ @Override
+ public VersioningIterator deepCopy(IteratorEnvironment env) {
+ VersioningIterator copy = new VersioningIterator();
+ copy.setSource(getSource().deepCopy(env));
+ copy.maxVersions = maxVersions;
+ return copy;
+ }
+
+ @Override
+ public void next() throws IOException {
+ if (numVersions >= maxVersions) {
+ skipRowColumn();
+ resetVersionCount();
+ return;
+ }
+
+ super.next();
+ if (getSource().hasTop()) {
+ if (getSource().getTopKey().equals(currentKey, PartialKey.ROW_COLFAM_COLQUAL_COLVIS)) {
+ numVersions++;
+ } else {
+ resetVersionCount();
+ }
+ }
+ }
+
+ @Override
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
+ // do not want to seek to the middle of a row
+ Range seekRange = IteratorUtil.maximizeStartKeyTimeStamp(range);
+ this.range = seekRange;
+ this.columnFamilies = columnFamilies;
+ this.inclusive = inclusive;
+
+ super.seek(seekRange, columnFamilies, inclusive);
+ resetVersionCount();
+
+ if (range.getStartKey() != null)
+ while (hasTop() && range.beforeStartKey(getTopKey()))
+ next();
+ }
+
+ private void resetVersionCount() {
+ if (super.hasTop())
+ currentKey.set(getSource().getTopKey());
+ numVersions = 1;
+ }
+
+ private void skipRowColumn() throws IOException {
+ Key keyToSkip = currentKey;
+ super.next();
+
+ int count = 0;
+ while (getSource().hasTop() && getSource().getTopKey().equals(keyToSkip, PartialKey.ROW_COLFAM_COLQUAL_COLVIS)) {
+ if (count < maxCount) {
+ // it is quicker to call next if we are close, but we never know if we are close
+ // so give next a try a few times
+ getSource().next();
+ count++;
+ } else {
+ reseek(keyToSkip.followingKey(PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
+ count = 0;
+ }
+ }
+ }
+
+ protected void reseek(Key key) throws IOException {
+ if (key == null)
+ return;
+ if (range.afterEndKey(key)) {
+ range = new Range(range.getEndKey(), true, range.getEndKey(), range.isEndKeyInclusive());
+ getSource().seek(range, columnFamilies, inclusive);
+ } else {
+ range = new Range(key, true, range.getEndKey(), range.isEndKeyInclusive());
+ getSource().seek(range, columnFamilies, inclusive);
+ }
+ }
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ super.init(source, options, env);
+ this.numVersions = 0;
+
+ String maxVerString = options.get("maxVersions");
+ if (maxVerString != null)
+ this.maxVersions = Integer.parseInt(maxVerString);
+ else
+ this.maxVersions = 1;
+
+ if (maxVersions < 1)
+ throw new IllegalArgumentException("maxVersions for versioning iterator must be >= 1");
+ }
+
+ @Override
+ public IteratorOptions describeOptions() {
+ return new IteratorOptions("vers", "The VersioningIterator keeps a fixed number of versions for each key", Collections.singletonMap("maxVersions",
+ "number of versions to keep for a particular key (with differing timestamps)"), null);
+ }
+
+ private static final String MAXVERSIONS_OPT = "maxVersions";
+
+ @Override
+ public boolean validateOptions(Map<String,String> options) {
+ int i;
+ try {
+ i = Integer.parseInt(options.get(MAXVERSIONS_OPT));
+ } catch (Exception e) {
+ throw new IllegalArgumentException("bad integer " + MAXVERSIONS_OPT + ":" + options.get(MAXVERSIONS_OPT));
+ }
+ if (i < 1)
+ throw new IllegalArgumentException(MAXVERSIONS_OPT + " for versioning iterator must be >= 1");
+ return true;
+ }
+
+ /**
+ * Encode the maximum number of versions to return onto the ScanIterator
- *
- * @param cfg
- * @param maxVersions
+ */
+ public static void setMaxVersions(IteratorSetting cfg, int maxVersions) {
+ if (maxVersions < 1)
+ throw new IllegalArgumentException(MAXVERSIONS_OPT + " for versioning iterator must be >= 1");
+ cfg.addOption(MAXVERSIONS_OPT, Integer.toString(maxVersions));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/security/SecurityUtil.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/SecurityUtil.java
index 672e784,0000000..65cb7ed
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/security/SecurityUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/SecurityUtil.java
@@@ -1,89 -1,0 +1,88 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.security;
+
+import java.io.IOException;
+import java.net.InetAddress;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.log4j.Logger;
+
+/**
+ *
+ */
+public class SecurityUtil {
+ private static final Logger log = Logger.getLogger(SecurityUtil.class);
+ private static final String ACCUMULO_HOME = "ACCUMULO_HOME", ACCUMULO_CONF_DIR = "ACCUMULO_CONF_DIR";
+ public static boolean usingKerberos = false;
+
+ /**
+ * This method is for logging a server in kerberos. If this is used in client code, it will fail unless run as the accumulo keytab's owner. Instead, use
+ * {@link #login(String, String)}
+ */
+ public static void serverLogin() {
+ @SuppressWarnings("deprecation")
+ AccumuloConfiguration acuConf = AccumuloConfiguration.getSiteConfiguration();
+ String keyTab = acuConf.get(Property.GENERAL_KERBEROS_KEYTAB);
+ if (keyTab == null || keyTab.length() == 0)
+ return;
+
+ usingKerberos = true;
+ if (keyTab.contains("$" + ACCUMULO_HOME) && System.getenv(ACCUMULO_HOME) != null)
+ keyTab = keyTab.replace("$" + ACCUMULO_HOME, System.getenv(ACCUMULO_HOME));
+
+ if (keyTab.contains("$" + ACCUMULO_CONF_DIR) && System.getenv(ACCUMULO_CONF_DIR) != null)
+ keyTab = keyTab.replace("$" + ACCUMULO_CONF_DIR, System.getenv(ACCUMULO_CONF_DIR));
+
+ String principalConfig = acuConf.get(Property.GENERAL_KERBEROS_PRINCIPAL);
+ if (principalConfig == null || principalConfig.length() == 0)
+ return;
+
+ if (login(principalConfig, keyTab)) {
+ try {
+ // This spawns a thread to periodically renew the logged in (accumulo) user
+ UserGroupInformation.getLoginUser();
+ } catch (IOException io) {
+ log.error("Error starting up renewal thread. This shouldn't be happenining.", io);
+ }
+ }
+ }
+
+ /**
+ * This will log in the given user in kerberos.
+ *
+ * @param principalConfig
+ * This is the principals name in the format NAME/HOST@REALM. {@link org.apache.hadoop.security.SecurityUtil#HOSTNAME_PATTERN} will automatically be
+ * replaced by the systems host name.
- * @param keyTabPath
+ * @return true if login succeeded, otherwise false
+ */
+ public static boolean login(String principalConfig, String keyTabPath) {
+ try {
+ String principalName = org.apache.hadoop.security.SecurityUtil.getServerPrincipal(principalConfig, InetAddress.getLocalHost().getCanonicalHostName());
+ if (keyTabPath != null && principalName != null && keyTabPath.length() != 0 && principalName.length() != 0) {
+ UserGroupInformation.loginUserFromKeytab(principalName, keyTabPath);
+ log.info("Succesfully logged in as user " + principalConfig);
+ return true;
+ }
+ } catch (IOException io) {
+ log.error("Error logging in user " + principalConfig + " using keytab at " + keyTabPath, io);
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModule.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModule.java
index fca7d22,0000000..40d3ab7
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModule.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModule.java
@@@ -1,110 -1,0 +1,108 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.security.crypto;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Map;
+
+/**
+ * Classes that obey this interface may be used to provide encrypting and decrypting streams to the rest of Accumulo. Classes that obey this interface may be
+ * configured as the crypto module by setting the property crypto.module.class in the accumulo-site.xml file.
+ *
+ * Note that this first iteration of this API is considered deprecated because we anticipate it changing in non-backwards compatible ways as we explore the
+ * requirements for encryption in Accumulo. So, your mileage is gonna vary a lot as we go forward.
+ *
+ */
+@Deprecated
+public interface CryptoModule {
+
+ public enum CryptoInitProperty {
+ ALGORITHM_NAME("algorithm.name"), CIPHER_SUITE("cipher.suite"), INITIALIZATION_VECTOR("initialization.vector"), PLAINTEXT_SESSION_KEY(
+ "plaintext.session.key");
+
+ private CryptoInitProperty(String name) {
+ key = name;
+ }
+
+ private String key;
+
+ public String getKey() {
+ return key;
+ }
+ }
+
+ /**
+ * Wraps an OutputStream in an encrypting OutputStream. The given map contains the settings for the cryptographic algorithm to use. <b>Callers of this method
+ * should expect that the given OutputStream will be written to before cryptographic writes occur.</b> These writes contain the cryptographic information used
+ * to encrypt the following bytes (these data include the initialization vector, encrypted session key, and so on). If writing arbitrarily to the underlying
+ * stream is not desirable, users should call the other flavor of getEncryptingOutputStream which accepts these data as parameters.
+ *
+ * @param out
+ * the OutputStream to wrap
+ * @param cryptoOpts
+ * the cryptographic parameters to use; specific string names to look for will depend on the various implementations
+ * @return an OutputStream that wraps the given parameter
- * @throws IOException
+ */
+ public OutputStream getEncryptingOutputStream(OutputStream out, Map<String,String> cryptoOpts) throws IOException;
+
+ /**
+ * Wraps an InputStream and returns a decrypting input stream. The given map contains the settings for the intended cryptographic operations, but implementors
+ * should take care to ensure that the crypto from the given input stream matches their expectations about what they will use to decrypt it, as the parameters
+ * may have changed. Also, care should be taken around transitioning between non-encrypting and encrypting streams; implementors should handle the case where
+ * the given input stream is <b>not</b> encrypted at all.
+ *
+ * It is expected that this version of getDecryptingInputStream is called in conjunction with the getEncryptingOutputStream from above. It should expect its
+ * input streams to contain the data written by getEncryptingOutputStream.
+ *
+ * @param in
+ * the InputStream to wrap
+ * @param cryptoOpts
+ * the cryptographic parameters to use; specific string names to look for will depend on the various implementations
+ * @return an InputStream that wraps the given parameter
- * @throws IOException
+ */
+ public InputStream getDecryptingInputStream(InputStream in, Map<String,String> cryptoOpts) throws IOException;
+
+ /**
+ * Wraps an OutputStream in an encrypting OutputStream. The given map contains the settings for the cryptographic algorithm to use. The cryptoInitParams map
+ * contains all the cryptographic details to construct a key (or keys), initialization vectors, etc. and use them to properly initialize the stream for
+ * writing. These initialization parameters must be persisted elsewhere, along with the cryptographic configuration (algorithm, mode, etc.), so that they may
+ * be read in at the time of reading the encrypted content.
+ *
+ * @param out
+ * the OutputStream to wrap
+ * @param conf
+ * the cryptographic algorithm configuration
+ * @param cryptoInitParams
+ * the initialization parameters for the algorithm, usually including initialization vector and session key
+ * @return a wrapped output stream
+ */
+ public OutputStream getEncryptingOutputStream(OutputStream out, Map<String,String> conf, Map<CryptoModule.CryptoInitProperty,Object> cryptoInitParams);
+
+ /**
+ * Wraps an InputStream and returns a decrypting input stream. The given map contains the settings for the intended cryptographic operations, but implementors
+ * should take care to ensure that the crypto from the given input stream matches their expectations about what they will use to decrypt it, as the parameters
+ * may have changed. Also, care should be taken around transitioning between non-encrypting and encrypting streams; implementors should handle the case where
+ * the given input stream is <b>not</b> encrypted at all.
+ *
+ * The cryptoInitParams contains all necessary information to properly initialize the given cipher, usually including things like initialization vector and
+ * secret key.
+ */
+ public InputStream getDecryptingInputStream(InputStream in, Map<String,String> cryptoOpts, Map<CryptoModule.CryptoInitProperty,Object> cryptoInitParams)
+ throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleFactory.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleFactory.java
index 2f03e02,0000000..956c961
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleFactory.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleFactory.java
@@@ -1,254 -1,0 +1,253 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.security.crypto;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Map;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.log4j.Logger;
+
+/**
+ * This factory module exists to assist other classes in loading crypto modules.
+ *
+ * @deprecated This feature is experimental and may go away in future versions.
+ */
+@Deprecated
+public class CryptoModuleFactory {
+
+ private static Logger log = Logger.getLogger(CryptoModuleFactory.class);
+
+ /**
+ * This method returns a crypto module based on settings in the given configuration parameter.
+ *
- * @param conf
+ * @return a class implementing the CryptoModule interface. It will *never* return null; rather, it will return a class which obeys the interface but makes no
+ * changes to the underlying data.
+ */
+ public static CryptoModule getCryptoModule(AccumuloConfiguration conf) {
+ String cryptoModuleClassname = conf.get(Property.CRYPTO_MODULE_CLASS);
+ return getCryptoModule(cryptoModuleClassname);
+ }
+
+ @SuppressWarnings({"rawtypes"})
+ public static CryptoModule getCryptoModule(String cryptoModuleClassname) {
+ log.debug(String.format("About to instantiate crypto module %s", cryptoModuleClassname));
+
+ if (cryptoModuleClassname.equals("NullCryptoModule")) {
+ return new NullCryptoModule();
+ }
+
+ CryptoModule cryptoModule = null;
+ Class cryptoModuleClazz = null;
+ try {
+ cryptoModuleClazz = AccumuloVFSClassLoader.loadClass(cryptoModuleClassname);
+ } catch (ClassNotFoundException e1) {
+ log.warn(String.format("Could not find configured crypto module \"%s\". NO ENCRYPTION WILL BE USED.", cryptoModuleClassname));
+ return new NullCryptoModule();
+ }
+
+ // Check if the given class implements the CryptoModule interface
+ Class[] interfaces = cryptoModuleClazz.getInterfaces();
+ boolean implementsCryptoModule = false;
+
+ for (Class clazz : interfaces) {
+ if (clazz.equals(CryptoModule.class)) {
+ implementsCryptoModule = true;
+ break;
+ }
+ }
+
+ if (!implementsCryptoModule) {
+ log.warn("Configured Accumulo crypto module \"%s\" does not implement the CryptoModule interface. NO ENCRYPTION WILL BE USED.");
+ return new NullCryptoModule();
+ } else {
+ try {
+ cryptoModule = (CryptoModule) cryptoModuleClazz.newInstance();
+
+ log.debug("Successfully instantiated crypto module");
+
+ } catch (InstantiationException e) {
+ log.warn(String.format("Got instantiation exception %s when instantiating crypto module \"%s\". NO ENCRYPTION WILL BE USED.", e.getCause().getClass()
+ .getCanonicalName(), cryptoModuleClassname));
+ log.warn(e.getCause());
+ return new NullCryptoModule();
+ } catch (IllegalAccessException e) {
+ log.warn(String.format("Got illegal access exception when trying to instantiate crypto module \"%s\". NO ENCRYPTION WILL BE USED.",
+ cryptoModuleClassname));
+ log.warn(e);
+ return new NullCryptoModule();
+ }
+ }
+ return cryptoModule;
+ }
+
+ public static SecretKeyEncryptionStrategy getSecretKeyEncryptionStrategy(AccumuloConfiguration conf) {
+ String className = conf.get(Property.CRYPTO_SECRET_KEY_ENCRYPTION_STRATEGY_CLASS);
+ return getSecretKeyEncryptionStrategy(className);
+ }
+
+ @SuppressWarnings("rawtypes")
+ public static SecretKeyEncryptionStrategy getSecretKeyEncryptionStrategy(String className) {
+ if (className == null || className.equals("NullSecretKeyEncryptionStrategy")) {
+ return new NullSecretKeyEncryptionStrategy();
+ }
+
+ SecretKeyEncryptionStrategy strategy = null;
+ Class keyEncryptionStrategyClazz = null;
+ try {
+ keyEncryptionStrategyClazz = AccumuloVFSClassLoader.loadClass(className);
+ } catch (ClassNotFoundException e1) {
+ log.warn(String.format("Could not find configured secret key encryption strategy \"%s\". NO ENCRYPTION WILL BE USED.", className));
+ return new NullSecretKeyEncryptionStrategy();
+ }
+
+ // Check if the given class implements the CryptoModule interface
+ Class[] interfaces = keyEncryptionStrategyClazz.getInterfaces();
+ boolean implementsSecretKeyStrategy = false;
+
+ for (Class clazz : interfaces) {
+ if (clazz.equals(SecretKeyEncryptionStrategy.class)) {
+ implementsSecretKeyStrategy = true;
+ break;
+ }
+ }
+
+ if (!implementsSecretKeyStrategy) {
+ log.warn("Configured Accumulo secret key encryption strategy \"%s\" does not implement the SecretKeyEncryptionStrategy interface. NO ENCRYPTION WILL BE USED.");
+ return new NullSecretKeyEncryptionStrategy();
+ } else {
+ try {
+ strategy = (SecretKeyEncryptionStrategy) keyEncryptionStrategyClazz.newInstance();
+
+ log.debug("Successfully instantiated secret key encryption strategy");
+
+ } catch (InstantiationException e) {
+ log.warn(String.format("Got instantiation exception %s when instantiating secret key encryption strategy \"%s\". NO ENCRYPTION WILL BE USED.", e
+ .getCause().getClass().getCanonicalName(), className));
+ log.warn(e.getCause());
+ return new NullSecretKeyEncryptionStrategy();
+ } catch (IllegalAccessException e) {
+ log.warn(String.format("Got illegal access exception when trying to instantiate secret key encryption strategy \"%s\". NO ENCRYPTION WILL BE USED.",
+ className));
+ log.warn(e);
+ return new NullSecretKeyEncryptionStrategy();
+ }
+ }
+
+ return strategy;
+ }
+
+ private static class NullSecretKeyEncryptionStrategy implements SecretKeyEncryptionStrategy {
+
+ @Override
+ public SecretKeyEncryptionStrategyContext encryptSecretKey(SecretKeyEncryptionStrategyContext context) {
+ context.setEncryptedSecretKey(context.getPlaintextSecretKey());
+ context.setOpaqueKeyEncryptionKeyID("");
+
+ return context;
+ }
+
+ @Override
+ public SecretKeyEncryptionStrategyContext decryptSecretKey(SecretKeyEncryptionStrategyContext context) {
+ context.setPlaintextSecretKey(context.getEncryptedSecretKey());
+
+ return context;
+ }
+
+ @Override
+ public SecretKeyEncryptionStrategyContext getNewContext() {
+ return new SecretKeyEncryptionStrategyContext() {
+
+ @Override
+ public byte[] getPlaintextSecretKey() {
+ return plaintextSecretKey;
+ }
+
+ @Override
+ public void setPlaintextSecretKey(byte[] plaintextSecretKey) {
+ this.plaintextSecretKey = plaintextSecretKey;
+ }
+
+ @Override
+ public byte[] getEncryptedSecretKey() {
+ return encryptedSecretKey;
+ }
+
+ @Override
+ public void setEncryptedSecretKey(byte[] encryptedSecretKey) {
+ this.encryptedSecretKey = encryptedSecretKey;
+ }
+
+ @Override
+ public String getOpaqueKeyEncryptionKeyID() {
+ return opaqueKeyEncryptionKeyID;
+ }
+
+ @Override
+ public void setOpaqueKeyEncryptionKeyID(String opaqueKeyEncryptionKeyID) {
+ this.opaqueKeyEncryptionKeyID = opaqueKeyEncryptionKeyID;
+ }
+
+ @Override
+ public Map<String,String> getContext() {
+ return context;
+ }
+
+ @Override
+ public void setContext(Map<String,String> context) {
+ this.context = context;
+ }
+
+ private byte[] plaintextSecretKey;
+ private byte[] encryptedSecretKey;
+ private String opaqueKeyEncryptionKeyID;
+ private Map<String,String> context;
+ };
+ }
+
+ }
+
+ private static class NullCryptoModule implements CryptoModule {
+
+ @Override
+ public OutputStream getEncryptingOutputStream(OutputStream out, Map<String,String> cryptoOpts) throws IOException {
+ return out;
+ }
+
+ @Override
+ public InputStream getDecryptingInputStream(InputStream in, Map<String,String> cryptoOpts) throws IOException {
+ return in;
+ }
+
+ @Override
+ public OutputStream getEncryptingOutputStream(OutputStream out, Map<String,String> conf, Map<CryptoInitProperty,Object> cryptoInitParams) {
+ return out;
+ }
+
+ @Override
+ public InputStream getDecryptingInputStream(InputStream in, Map<String,String> cryptoOpts, Map<CryptoInitProperty,Object> cryptoInitParams)
+ throws IOException {
+ return in;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/test/java/org/apache/accumulo/core/client/impl/ScannerOptionsTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/impl/ScannerOptionsTest.java
index 07bd518,0000000..463822a
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/impl/ScannerOptionsTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/impl/ScannerOptionsTest.java
@@@ -1,59 -1,0 +1,57 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.impl;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.iterators.DebugIterator;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.junit.Test;
+
+/**
+ * Test that scanner options are set/unset correctly
+ */
+public class ScannerOptionsTest {
+
+ /**
+ * Test that you properly add and remove iterators from a scanner
- *
- * @throws Throwable
+ */
+ @Test
+ public void testAddRemoveIterator() throws Throwable {
+ ScannerOptions options = new ScannerOptions();
+ options.addScanIterator(new IteratorSetting(1, "NAME", WholeRowIterator.class));
+ assertEquals(1, options.serverSideIteratorList.size());
+ options.removeScanIterator("NAME");
+ assertEquals(0, options.serverSideIteratorList.size());
+ }
+
+ @Test
+ public void testIteratorConflict() {
+ ScannerOptions options = new ScannerOptions();
+ options.addScanIterator(new IteratorSetting(1, "NAME", DebugIterator.class));
+ try {
+ options.addScanIterator(new IteratorSetting(2, "NAME", DebugIterator.class));
+ fail();
+ } catch (IllegalArgumentException e) {}
+ try {
+ options.addScanIterator(new IteratorSetting(1, "NAME2", DebugIterator.class));
+ fail();
+ } catch (IllegalArgumentException e) {}
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
index 4f527e1,0000000..3ec9bb1
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
@@@ -1,284 -1,0 +1,280 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapred;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.lib.NullOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.Test;
+
+public class AccumuloInputFormatTest {
+
+ private static final String PREFIX = AccumuloInputFormatTest.class.getSimpleName();
+ private static final String INSTANCE_NAME = PREFIX + "_mapred_instance";
+ private static final String TEST_TABLE_1 = PREFIX + "_mapred_table_1";
+
+ /**
+ * Check that the iterator configuration is getting stored in the Job conf correctly.
- *
- * @throws IOException
+ */
+ @Test
+ public void testSetIterator() throws IOException {
+ JobConf job = new JobConf();
+
+ IteratorSetting is = new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator");
+ AccumuloInputFormat.addIterator(job, is);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ is.write(new DataOutputStream(baos));
+ String iterators = job.get("AccumuloInputFormat.ScanOpts.Iterators");
+ assertEquals(new String(Base64.encodeBase64(baos.toByteArray())), iterators);
+ }
+
+ @Test
+ public void testAddIterator() throws IOException {
+ JobConf job = new JobConf();
+
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", WholeRowIterator.class));
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
+ IteratorSetting iter = new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator");
+ iter.addOption("v1", "1");
+ iter.addOption("junk", "\0omg:!\\xyzzy");
+ AccumuloInputFormat.addIterator(job, iter);
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+
+ // Check the list size
+ assertTrue(list.size() == 3);
+
+ // Walk the list and make sure our settings are correct
+ IteratorSetting setting = list.get(0);
+ assertEquals(1, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.user.WholeRowIterator", setting.getIteratorClass());
+ assertEquals("WholeRow", setting.getName());
+ assertEquals(0, setting.getOptions().size());
+
+ setting = list.get(1);
+ assertEquals(2, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
+ assertEquals("Versions", setting.getName());
+ assertEquals(0, setting.getOptions().size());
+
+ setting = list.get(2);
+ assertEquals(3, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
+ assertEquals("Count", setting.getName());
+ assertEquals(2, setting.getOptions().size());
+ assertEquals("1", setting.getOptions().get("v1"));
+ assertEquals("\0omg:!\\xyzzy", setting.getOptions().get("junk"));
+ }
+
+ /**
+ * Test adding iterator options where the keys and values contain both the FIELD_SEPARATOR character (':') and ITERATOR_SEPARATOR (',') characters. There
+ * should be no exceptions thrown when trying to parse these types of option entries.
+ *
+ * This test makes sure that the expected raw values, as appears in the Job, are equal to what's expected.
+ */
+ @Test
+ public void testIteratorOptionEncoding() throws Throwable {
+ String key = "colon:delimited:key";
+ String value = "comma,delimited,value";
+ IteratorSetting someSetting = new IteratorSetting(1, "iterator", "Iterator.class");
+ someSetting.addOption(key, value);
+ JobConf job = new JobConf();
+ AccumuloInputFormat.addIterator(job, someSetting);
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+ assertEquals(1, list.size());
+ assertEquals(1, list.get(0).getOptions().size());
+ assertEquals(list.get(0).getOptions().get(key), value);
+
+ someSetting.addOption(key + "2", value);
+ someSetting.setPriority(2);
+ someSetting.setName("it2");
+ AccumuloInputFormat.addIterator(job, someSetting);
+ list = AccumuloInputFormat.getIterators(job);
+ assertEquals(2, list.size());
+ assertEquals(1, list.get(0).getOptions().size());
+ assertEquals(list.get(0).getOptions().get(key), value);
+ assertEquals(2, list.get(1).getOptions().size());
+ assertEquals(list.get(1).getOptions().get(key), value);
+ assertEquals(list.get(1).getOptions().get(key + "2"), value);
+ }
+
+ /**
+ * Test getting iterator settings for multiple iterators set
- *
- * @throws IOException
+ */
+ @Test
+ public void testGetIteratorSettings() throws IOException {
+ JobConf job = new JobConf();
+
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator"));
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator"));
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+
+ // Check the list size
+ assertTrue(list.size() == 3);
+
+ // Walk the list and make sure our settings are correct
+ IteratorSetting setting = list.get(0);
+ assertEquals(1, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.WholeRowIterator", setting.getIteratorClass());
+ assertEquals("WholeRow", setting.getName());
+
+ setting = list.get(1);
+ assertEquals(2, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
+ assertEquals("Versions", setting.getName());
+
+ setting = list.get(2);
+ assertEquals(3, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
+ assertEquals("Count", setting.getName());
+
+ }
+
+ @Test
+ public void testSetRegex() throws IOException {
+ JobConf job = new JobConf();
+
+ String regex = ">\"*%<>\'\\";
+
+ IteratorSetting is = new IteratorSetting(50, regex, RegExFilter.class);
+ RegExFilter.setRegexs(is, regex, null, null, null, false);
+ AccumuloInputFormat.addIterator(job, is);
+
+ assertTrue(regex.equals(AccumuloInputFormat.getIterators(job).get(0).getName()));
+ }
+
+ private static AssertionError e1 = null;
+ private static AssertionError e2 = null;
+
+ private static class MRTester extends Configured implements Tool {
+ private static class TestMapper implements Mapper<Key,Value,Key,Value> {
+ Key key = null;
+ int count = 0;
+
+ @Override
+ public void map(Key k, Value v, OutputCollector<Key,Value> output, Reporter reporter) throws IOException {
+ try {
+ if (key != null)
+ assertEquals(key.getRow().toString(), new String(v.get()));
+ assertEquals(k.getRow(), new Text(String.format("%09x", count + 1)));
+ assertEquals(new String(v.get()), String.format("%09x", count));
+ } catch (AssertionError e) {
+ e1 = e;
+ }
+ key = new Key(k);
+ count++;
+ }
+
+ @Override
+ public void configure(JobConf job) {}
+
+ @Override
+ public void close() throws IOException {
+ try {
+ assertEquals(100, count);
+ } catch (AssertionError e) {
+ e2 = e;
+ }
+ }
+
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+
+ if (args.length != 3) {
+ throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <user> <pass> <table>");
+ }
+
+ String user = args[0];
+ String pass = args[1];
+ String table = args[2];
+
+ JobConf job = new JobConf(getConf());
+ job.setJarByClass(this.getClass());
+
+ job.setInputFormat(AccumuloInputFormat.class);
+
+ AccumuloInputFormat.setConnectorInfo(job, user, new PasswordToken(pass));
+ AccumuloInputFormat.setInputTableName(job, table);
+ AccumuloInputFormat.setMockInstance(job, INSTANCE_NAME);
+
+ job.setMapperClass(TestMapper.class);
+ job.setMapOutputKeyClass(Key.class);
+ job.setMapOutputValueClass(Value.class);
+ job.setOutputFormat(NullOutputFormat.class);
+
+ job.setNumReduceTasks(0);
+
+ return JobClient.runJob(job).isSuccessful() ? 0 : 1;
+ }
+
+ public static void main(String[] args) throws Exception {
+ assertEquals(0, ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args));
+ }
+ }
+
+ @Test
+ public void testMap() throws Exception {
+ MockInstance mockInstance = new MockInstance(INSTANCE_NAME);
+ Connector c = mockInstance.getConnector("root", new PasswordToken(""));
+ c.tableOperations().create(TEST_TABLE_1);
+ BatchWriter bw = c.createBatchWriter(TEST_TABLE_1, new BatchWriterConfig());
+ for (int i = 0; i < 100; i++) {
+ Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
+ m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
+ bw.addMutation(m);
+ }
+ bw.close();
+
+ MRTester.main(new String[] {"root", "", TEST_TABLE_1});
+ assertNull(e1);
+ assertNull(e2);
+ }
+}
[48/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftTransportPool.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/ThriftTransportPool.java
index f123289,0000000..d2113fb
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftTransportPool.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftTransportPool.java
@@@ -1,671 -1,0 +1,683 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.impl;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.security.SecurityPermission;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.TTimeoutTransport;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.log4j.Logger;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+
+public class ThriftTransportPool {
+ private static SecurityPermission TRANSPORT_POOL_PERMISSION = new SecurityPermission("transportPoolPermission");
+
+ private static final Random random = new Random();
+ private long killTime = 1000 * 3;
+
+ private Map<ThriftTransportKey,List<CachedConnection>> cache = new HashMap<ThriftTransportKey,List<CachedConnection>>();
+ private Map<ThriftTransportKey,Long> errorCount = new HashMap<ThriftTransportKey,Long>();
+ private Map<ThriftTransportKey,Long> errorTime = new HashMap<ThriftTransportKey,Long>();
+ private Set<ThriftTransportKey> serversWarnedAbout = new HashSet<ThriftTransportKey>();
+
+ private CountDownLatch closerExitLatch;
+
+ private static final Logger log = Logger.getLogger(ThriftTransportPool.class);
+
+ private static final Long ERROR_THRESHOLD = 20l;
+ private static final int STUCK_THRESHOLD = 2 * 60 * 1000;
+
+ private static class CachedConnection {
+
+ public CachedConnection(CachedTTransport t) {
+ this.transport = t;
+ }
+
+ void setReserved(boolean reserved) {
+ this.transport.setReserved(reserved);
+ }
+
+ boolean isReserved() {
+ return this.transport.reserved;
+ }
+
+ CachedTTransport transport;
+
+ long lastReturnTime;
+ }
+
+ public static class TransportPoolShutdownException extends RuntimeException {
+ private static final long serialVersionUID = 1L;
+ }
+
+ private static class Closer implements Runnable {
+ final ThriftTransportPool pool;
+ private CountDownLatch closerExitLatch;
+
+ public Closer(ThriftTransportPool pool, CountDownLatch closerExitLatch) {
+ this.pool = pool;
+ this.closerExitLatch = closerExitLatch;
+ }
+
+ private void closeConnections() {
+ while (true) {
+
+ ArrayList<CachedConnection> connectionsToClose = new ArrayList<CachedConnection>();
+
+ synchronized (pool) {
+ for (List<CachedConnection> ccl : pool.getCache().values()) {
+ Iterator<CachedConnection> iter = ccl.iterator();
+ while (iter.hasNext()) {
+ CachedConnection cachedConnection = iter.next();
+
+ if (!cachedConnection.isReserved() && System.currentTimeMillis() - cachedConnection.lastReturnTime > pool.killTime) {
+ connectionsToClose.add(cachedConnection);
+ iter.remove();
+ }
+ }
+ }
+
+ for (List<CachedConnection> ccl : pool.getCache().values()) {
+ for (CachedConnection cachedConnection : ccl) {
+ cachedConnection.transport.checkForStuckIO(STUCK_THRESHOLD);
+ }
+ }
+
+ Iterator<Entry<ThriftTransportKey,Long>> iter = pool.errorTime.entrySet().iterator();
+ while (iter.hasNext()) {
+ Entry<ThriftTransportKey,Long> entry = iter.next();
+ long delta = System.currentTimeMillis() - entry.getValue();
+ if (delta >= STUCK_THRESHOLD) {
+ pool.errorCount.remove(entry.getKey());
+ iter.remove();
+ }
+ }
+ }
+
+ // close connections outside of sync block
+ for (CachedConnection cachedConnection : connectionsToClose) {
+ cachedConnection.transport.close();
+ }
+
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
++ @Override
+ public void run() {
+ try {
+ closeConnections();
+ } catch (TransportPoolShutdownException e) {
+ } finally {
+ closerExitLatch.countDown();
+ }
+ }
+ }
+
+ static class CachedTTransport extends TTransport {
+
+ private ThriftTransportKey cacheKey;
+ private TTransport wrappedTransport;
+ private boolean sawError = false;
+
+ private volatile String ioThreadName = null;
+ private volatile long ioStartTime = 0;
+ private volatile boolean reserved = false;
+
+ private String stuckThreadName = null;
+
+ int ioCount = 0;
+ int lastIoCount = -1;
+
+ private void sawError(Exception e) {
+ sawError = true;
+ }
+
+ final void setReserved(boolean reserved) {
+ this.reserved = reserved;
+ if (reserved) {
+ ioThreadName = Thread.currentThread().getName();
+ ioCount = 0;
+ lastIoCount = -1;
+ } else {
+ if ((ioCount & 1) == 1) {
+ // connection unreserved, but it seems io may still be
+ // happening
+ log.warn("Connection returned to thrift connection pool that may still be in use " + ioThreadName + " " + Thread.currentThread().getName(),
+ new Exception());
+ }
+
+ ioCount = 0;
+ lastIoCount = -1;
+ ioThreadName = null;
+ }
+ checkForStuckIO(STUCK_THRESHOLD);
+ }
+
+ final void checkForStuckIO(long threshold) {
+ /*
+ * checking for stuck io needs to be light weight.
+ *
+ * Tried to call System.currentTimeMillis() and Thread.currentThread() before every io operation.... this dramatically slowed things down. So switched to
+ * incrementing a counter before and after each io operation.
+ */
+
+ if ((ioCount & 1) == 1) {
+ // when ioCount is odd, it means I/O is currently happening
+ if (ioCount == lastIoCount) {
+ // still doing same I/O operation as last time this
+ // functions was called
+ long delta = System.currentTimeMillis() - ioStartTime;
+ if (delta >= threshold && stuckThreadName == null) {
+ stuckThreadName = ioThreadName;
+ log.warn("Thread \"" + ioThreadName + "\" stuck on IO to " + cacheKey + " for at least " + delta + " ms");
+ }
+ } else {
+ // remember this ioCount and the time we saw it, need to see
+ // if it changes
+ lastIoCount = ioCount;
+ ioStartTime = System.currentTimeMillis();
+
+ if (stuckThreadName != null) {
+ // doing I/O, but ioCount changed so no longer stuck
+ log.info("Thread \"" + stuckThreadName + "\" no longer stuck on IO to " + cacheKey + " sawError = " + sawError);
+ stuckThreadName = null;
+ }
+ }
+ } else {
+ // I/O is not currently happening
+ if (stuckThreadName != null) {
+ // no longer stuck, and was stuck in the past
+ log.info("Thread \"" + stuckThreadName + "\" no longer stuck on IO to " + cacheKey + " sawError = " + sawError);
+ stuckThreadName = null;
+ }
+ }
+ }
+
+ public CachedTTransport(TTransport transport, ThriftTransportKey cacheKey2) {
+ this.wrappedTransport = transport;
+ this.cacheKey = cacheKey2;
+ }
+
++ @Override
+ public boolean isOpen() {
+ return wrappedTransport.isOpen();
+ }
+
++ @Override
+ public void open() throws TTransportException {
+ try {
+ ioCount++;
+ wrappedTransport.open();
+ } catch (TTransportException tte) {
+ sawError(tte);
+ throw tte;
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public int read(byte[] arg0, int arg1, int arg2) throws TTransportException {
+ try {
+ ioCount++;
+ return wrappedTransport.read(arg0, arg1, arg2);
+ } catch (TTransportException tte) {
+ sawError(tte);
+ throw tte;
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public int readAll(byte[] arg0, int arg1, int arg2) throws TTransportException {
+ try {
+ ioCount++;
+ return wrappedTransport.readAll(arg0, arg1, arg2);
+ } catch (TTransportException tte) {
+ sawError(tte);
+ throw tte;
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public void write(byte[] arg0, int arg1, int arg2) throws TTransportException {
+ try {
+ ioCount++;
+ wrappedTransport.write(arg0, arg1, arg2);
+ } catch (TTransportException tte) {
+ sawError(tte);
+ throw tte;
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public void write(byte[] arg0) throws TTransportException {
+ try {
+ ioCount++;
+ wrappedTransport.write(arg0);
+ } catch (TTransportException tte) {
+ sawError(tte);
+ throw tte;
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public void close() {
+ try {
+ ioCount++;
+ wrappedTransport.close();
+ } finally {
+ ioCount++;
+ }
+
+ }
+
++ @Override
+ public void flush() throws TTransportException {
+ try {
+ ioCount++;
+ wrappedTransport.flush();
+ } catch (TTransportException tte) {
+ sawError(tte);
+ throw tte;
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public boolean peek() {
+ try {
+ ioCount++;
+ return wrappedTransport.peek();
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public byte[] getBuffer() {
+ try {
+ ioCount++;
+ return wrappedTransport.getBuffer();
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public int getBufferPosition() {
+ try {
+ ioCount++;
+ return wrappedTransport.getBufferPosition();
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public int getBytesRemainingInBuffer() {
+ try {
+ ioCount++;
+ return wrappedTransport.getBytesRemainingInBuffer();
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public void consumeBuffer(int len) {
+ try {
+ ioCount++;
+ wrappedTransport.consumeBuffer(len);
+ } finally {
+ ioCount++;
+ }
+ }
+
+ public ThriftTransportKey getCacheKey() {
+ return cacheKey;
+ }
+
+ }
+
+ private ThriftTransportPool() {}
+
+ public TTransport getTransport(String location, int port) throws TTransportException {
+ return getTransport(location, port, 0);
+ }
+
+ public TTransport getTransportWithDefaultTimeout(InetSocketAddress addr, AccumuloConfiguration conf) throws TTransportException {
+ return getTransport(addr.getAddress().getHostAddress(), addr.getPort(), conf.getTimeInMillis(Property.GENERAL_RPC_TIMEOUT));
+ }
+
+ public TTransport getTransport(InetSocketAddress addr, long timeout) throws TTransportException {
+ return getTransport(addr.getAddress().getHostAddress(), addr.getPort(), timeout);
+ }
+
+ public TTransport getTransportWithDefaultTimeout(String location, int port, AccumuloConfiguration conf) throws TTransportException {
+ return getTransport(location, port, conf.getTimeInMillis(Property.GENERAL_RPC_TIMEOUT));
+ }
+
+ Pair<String,TTransport> getAnyTransport(List<ThriftTransportKey> servers, boolean preferCachedConnection) throws TTransportException {
+
+ servers = new ArrayList<ThriftTransportKey>(servers);
+
+ if (preferCachedConnection) {
+ HashSet<ThriftTransportKey> serversSet = new HashSet<ThriftTransportKey>(servers);
+
+ synchronized (this) {
+
+ // randomly pick a server from the connection cache
+ serversSet.retainAll(getCache().keySet());
+
+ if (serversSet.size() > 0) {
+ ArrayList<ThriftTransportKey> cachedServers = new ArrayList<ThriftTransportKey>(serversSet);
+ Collections.shuffle(cachedServers, random);
+
+ for (ThriftTransportKey ttk : cachedServers) {
+ for (CachedConnection cachedConnection : getCache().get(ttk)) {
+ if (!cachedConnection.isReserved()) {
+ cachedConnection.setReserved(true);
+ if (log.isTraceEnabled())
+ log.trace("Using existing connection to " + ttk.getLocation() + ":" + ttk.getPort());
+ return new Pair<String,TTransport>(ttk.getLocation() + ":" + ttk.getPort(), cachedConnection.transport);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ int retryCount = 0;
+ while (servers.size() > 0 && retryCount < 10) {
+ int index = random.nextInt(servers.size());
+ ThriftTransportKey ttk = servers.get(index);
+
+ if (!preferCachedConnection) {
+ synchronized (this) {
+ List<CachedConnection> cachedConnList = getCache().get(ttk);
+ if (cachedConnList != null) {
+ for (CachedConnection cachedConnection : cachedConnList) {
+ if (!cachedConnection.isReserved()) {
+ cachedConnection.setReserved(true);
+ if (log.isTraceEnabled())
+ log.trace("Using existing connection to " + ttk.getLocation() + ":" + ttk.getPort() + " timeout " + ttk.getTimeout());
+ return new Pair<String,TTransport>(ttk.getLocation() + ":" + ttk.getPort(), cachedConnection.transport);
+ }
+ }
+ }
+ }
+ }
+
+ try {
+ return new Pair<String,TTransport>(ttk.getLocation() + ":" + ttk.getPort(), createNewTransport(ttk));
+ } catch (TTransportException tte) {
+ log.debug("Failed to connect to " + servers.get(index), tte);
+ servers.remove(index);
+ retryCount++;
+ }
+ }
+
+ throw new TTransportException("Failed to connect to a server");
+ }
+
+ public TTransport getTransport(String location, int port, long milliseconds) throws TTransportException {
+ return getTransport(new ThriftTransportKey(location, port, milliseconds));
+ }
+
+ private TTransport getTransport(ThriftTransportKey cacheKey) throws TTransportException {
+ synchronized (this) {
+ // atomically reserve location if it exist in cache
+ List<CachedConnection> ccl = getCache().get(cacheKey);
+
+ if (ccl == null) {
+ ccl = new LinkedList<CachedConnection>();
+ getCache().put(cacheKey, ccl);
+ }
+
+ for (CachedConnection cachedConnection : ccl) {
+ if (!cachedConnection.isReserved()) {
+ cachedConnection.setReserved(true);
+ if (log.isTraceEnabled())
+ log.trace("Using existing connection to " + cacheKey.getLocation() + ":" + cacheKey.getPort());
+ return cachedConnection.transport;
+ }
+ }
+ }
+
+ return createNewTransport(cacheKey);
+ }
+
+ private TTransport createNewTransport(ThriftTransportKey cacheKey) throws TTransportException {
+ TTransport transport;
+ if (cacheKey.getTimeout() == 0) {
+ transport = AddressUtil.createTSocket(cacheKey.getLocation(), cacheKey.getPort());
+ } else {
+ try {
+ transport = TTimeoutTransport.create(AddressUtil.parseAddress(cacheKey.getLocation(), cacheKey.getPort()), cacheKey.getTimeout());
+ } catch (IOException ex) {
+ throw new TTransportException(ex);
+ }
+ }
+ transport = ThriftUtil.transportFactory().getTransport(transport);
+ transport.open();
+
+ if (log.isTraceEnabled())
+ log.trace("Creating new connection to connection to " + cacheKey.getLocation() + ":" + cacheKey.getPort());
+
+ CachedTTransport tsc = new CachedTTransport(transport, cacheKey);
+
+ CachedConnection cc = new CachedConnection(tsc);
+ cc.setReserved(true);
+
+ try {
+ synchronized (this) {
+ List<CachedConnection> ccl = getCache().get(cacheKey);
+
+ if (ccl == null) {
+ ccl = new LinkedList<CachedConnection>();
+ getCache().put(cacheKey, ccl);
+ }
+
+ ccl.add(cc);
+ }
+ } catch (TransportPoolShutdownException e) {
+ cc.transport.close();
+ throw e;
+ }
+ return cc.transport;
+ }
+
+ public void returnTransport(TTransport tsc) {
+ if (tsc == null) {
+ return;
+ }
+
+ boolean existInCache = false;
+ CachedTTransport ctsc = (CachedTTransport) tsc;
+
+ ArrayList<CachedConnection> closeList = new ArrayList<ThriftTransportPool.CachedConnection>();
+
+ synchronized (this) {
+ List<CachedConnection> ccl = getCache().get(ctsc.getCacheKey());
+ for (Iterator<CachedConnection> iterator = ccl.iterator(); iterator.hasNext();) {
+ CachedConnection cachedConnection = iterator.next();
+ if (cachedConnection.transport == tsc) {
+ if (ctsc.sawError) {
+ closeList.add(cachedConnection);
+ iterator.remove();
+
+ if (log.isTraceEnabled())
+ log.trace("Returned connection had error " + ctsc.getCacheKey());
+
+ Long ecount = errorCount.get(ctsc.getCacheKey());
+ if (ecount == null)
+ ecount = 0l;
+ ecount++;
+ errorCount.put(ctsc.getCacheKey(), ecount);
+
+ Long etime = errorTime.get(ctsc.getCacheKey());
+ if (etime == null) {
+ errorTime.put(ctsc.getCacheKey(), System.currentTimeMillis());
+ }
+
+ if (ecount >= ERROR_THRESHOLD && !serversWarnedAbout.contains(ctsc.getCacheKey())) {
+ log.warn("Server " + ctsc.getCacheKey() + " had " + ecount + " failures in a short time period, will not complain anymore ");
+ serversWarnedAbout.add(ctsc.getCacheKey());
+ }
+
+ cachedConnection.setReserved(false);
+
+ } else {
+
+ if (log.isTraceEnabled())
+ log.trace("Returned connection " + ctsc.getCacheKey() + " ioCount : " + cachedConnection.transport.ioCount);
+
+ cachedConnection.lastReturnTime = System.currentTimeMillis();
+ cachedConnection.setReserved(false);
+ }
+ existInCache = true;
+ break;
+ }
+ }
+
+ // remove all unreserved cached connection when a sever has an error, not just the connection that was returned
+ if (ctsc.sawError) {
+ for (Iterator<CachedConnection> iterator = ccl.iterator(); iterator.hasNext();) {
+ CachedConnection cachedConnection = iterator.next();
+ if (!cachedConnection.isReserved()) {
+ closeList.add(cachedConnection);
+ iterator.remove();
+ }
+ }
+ }
+ }
+
+ // close outside of sync block
+ for (CachedConnection cachedConnection : closeList) {
+ try {
+ cachedConnection.transport.close();
+ } catch (Exception e) {
+ log.debug("Failed to close connection w/ errors", e);
+ }
+ }
+
+ if (!existInCache) {
+ log.warn("Returned tablet server connection to cache that did not come from cache");
+ // close outside of sync block
+ tsc.close();
+ }
+ }
+
+ /**
+ * Set the time after which idle connections should be closed
- *
- * @param time
+ */
+ public synchronized void setIdleTime(long time) {
+ this.killTime = time;
+ log.debug("Set thrift transport pool idle time to " + time);
+ }
+
+ private static ThriftTransportPool instance = new ThriftTransportPool();
+ private static final AtomicBoolean daemonStarted = new AtomicBoolean(false);
+
+ public static ThriftTransportPool getInstance() {
+ SecurityManager sm = System.getSecurityManager();
+ if (sm != null) {
+ sm.checkPermission(TRANSPORT_POOL_PERMISSION);
+ }
+
+ if (daemonStarted.compareAndSet(false, true)) {
+ CountDownLatch closerExitLatch = new CountDownLatch(1);
+ new Daemon(new Closer(instance, closerExitLatch), "Thrift Connection Pool Checker").start();
+ instance.setCloserExitLatch(closerExitLatch);
+ }
+ return instance;
+ }
+
+ private synchronized void setCloserExitLatch(CountDownLatch closerExitLatch) {
+ this.closerExitLatch = closerExitLatch;
+ }
+
+ public void shutdown() {
+ synchronized (this) {
+ if (cache == null)
+ return;
+
+ // close any connections in the pool... even ones that are in use
+ for (List<CachedConnection> ccl : getCache().values()) {
+ Iterator<CachedConnection> iter = ccl.iterator();
+ while (iter.hasNext()) {
+ CachedConnection cc = iter.next();
+ try {
+ cc.transport.close();
+ } catch (Exception e) {
+ log.debug("Error closing transport during shutdown", e);
+ }
+ }
+ }
+
+ // this will render the pool unusable and cause the background thread to exit
+ this.cache = null;
+ }
+
+ try {
+ closerExitLatch.await();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private Map<ThriftTransportKey,List<CachedConnection>> getCache() {
+ if (cache == null)
+ throw new TransportPoolShutdownException();
+ return cache;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
index ee4aca5,0000000..d7be37c
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
@@@ -1,511 -1,0 +1,510 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapred;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.mapreduce.lib.util.OutputConfigurator;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.SecurityErrorCode;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.OutputFormat;
+import org.apache.hadoop.mapred.RecordWriter;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.util.Progressable;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * This class allows MapReduce jobs to use Accumulo as the sink for data. This {@link OutputFormat} accepts keys and values of type {@link Text} (for a table
+ * name) and {@link Mutation} from the Map and Reduce functions.
+ *
+ * The user must specify the following via static configurator methods:
+ *
+ * <ul>
+ * <li>{@link AccumuloOutputFormat#setConnectorInfo(JobConf, String, AuthenticationToken)}
+ * <li>{@link AccumuloOutputFormat#setZooKeeperInstance(JobConf, String, String)} OR {@link AccumuloOutputFormat#setMockInstance(JobConf, String)}
+ * </ul>
+ *
+ * Other static methods are optional.
+ */
+public class AccumuloOutputFormat implements OutputFormat<Text,Mutation> {
+
+ private static final Class<?> CLASS = AccumuloOutputFormat.class;
+ protected static final Logger log = Logger.getLogger(CLASS);
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
+ * conversion to a string, and is not intended to be secure.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission if {@link #setCreateTables(JobConf, boolean)} is set to true)
+ * @param token
+ * the user's password
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public static void setConnectorInfo(JobConf job, String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ OutputConfigurator.setConnectorInfo(CLASS, job, principal, token);
+ }
+
+ /**
+ * Determines if the connector has been configured.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the connector has been configured, false otherwise
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static Boolean isConnectorInfoSet(JobConf job) {
+ return OutputConfigurator.isConnectorInfoSet(CLASS, job);
+ }
+
+ /**
+ * Gets the principal from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static String getPrincipal(JobConf job) {
+ return OutputConfigurator.getPrincipal(CLASS, job);
+ }
+
+ /**
+ * Gets the serialized token class from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static String getTokenClass(JobConf job) {
+ return OutputConfigurator.getTokenClass(CLASS, job);
+ }
+
+ /**
+ * Gets the password from the configuration. WARNING: The password is stored in the Configuration and shared with all MapReduce tasks; It is BASE64 encoded to
+ * provide a charset safe conversion to a string, and is not intended to be secure.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the decoded user password
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static byte[] getToken(JobConf job) {
+ return OutputConfigurator.getToken(CLASS, job);
+ }
+
+ /**
+ * Configures a {@link ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @param zooKeepers
+ * a comma-separated list of zookeeper servers
+ * @since 1.5.0
+ */
+ public static void setZooKeeperInstance(JobConf job, String instanceName, String zooKeepers) {
+ OutputConfigurator.setZooKeeperInstance(CLASS, job, instanceName, zooKeepers);
+ }
+
+ /**
+ * Configures a {@link MockInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @since 1.5.0
+ */
+ public static void setMockInstance(JobConf job, String instanceName) {
+ OutputConfigurator.setMockInstance(CLASS, job, instanceName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link Instance} based on the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return an Accumulo instance
+ * @since 1.5.0
+ * @see #setZooKeeperInstance(JobConf, String, String)
+ * @see #setMockInstance(JobConf, String)
+ */
+ protected static Instance getInstance(JobConf job) {
+ return OutputConfigurator.getInstance(CLASS, job);
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param level
+ * the logging level
+ * @since 1.5.0
+ */
+ public static void setLogLevel(JobConf job, Level level) {
+ OutputConfigurator.setLogLevel(CLASS, job, level);
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the log level
+ * @since 1.5.0
+ * @see #setLogLevel(JobConf, Level)
+ */
+ protected static Level getLogLevel(JobConf job) {
+ return OutputConfigurator.getLogLevel(CLASS, job);
+ }
+
+ /**
+ * Sets the default table name to use if one emits a null in place of a table name for a given mutation. Table names can only be alpha-numeric and
+ * underscores.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.5.0
+ */
+ public static void setDefaultTableName(JobConf job, String tableName) {
+ OutputConfigurator.setDefaultTableName(CLASS, job, tableName);
+ }
+
+ /**
+ * Gets the default table name from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the default table name
+ * @since 1.5.0
+ * @see #setDefaultTableName(JobConf, String)
+ */
+ protected static String getDefaultTableName(JobConf job) {
+ return OutputConfigurator.getDefaultTableName(CLASS, job);
+ }
+
+ /**
+ * Sets the configuration for for the job's {@link BatchWriter} instances. If not set, a new {@link BatchWriterConfig}, with sensible built-in defaults is
+ * used. Setting the configuration multiple times overwrites any previous configuration.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param bwConfig
+ * the configuration for the {@link BatchWriter}
+ * @since 1.5.0
+ */
+ public static void setBatchWriterOptions(JobConf job, BatchWriterConfig bwConfig) {
+ OutputConfigurator.setBatchWriterOptions(CLASS, job, bwConfig);
+ }
+
+ /**
+ * Gets the {@link BatchWriterConfig} settings.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the configuration object
+ * @since 1.5.0
+ * @see #setBatchWriterOptions(JobConf, BatchWriterConfig)
+ */
+ protected static BatchWriterConfig getBatchWriterOptions(JobConf job) {
+ return OutputConfigurator.getBatchWriterOptions(CLASS, job);
+ }
+
+ /**
+ * Sets the directive to create new tables, as necessary. Table names can only be alpha-numeric and underscores.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setCreateTables(JobConf job, boolean enableFeature) {
+ OutputConfigurator.setCreateTables(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether tables are permitted to be created as needed.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the feature is disabled, false otherwise
+ * @since 1.5.0
+ * @see #setCreateTables(JobConf, boolean)
+ */
+ protected static Boolean canCreateTables(JobConf job) {
+ return OutputConfigurator.canCreateTables(CLASS, job);
+ }
+
+ /**
+ * Sets the directive to use simulation mode for this job. In simulation mode, no output is produced. This is useful for testing.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setSimulationMode(JobConf job, boolean enableFeature) {
+ OutputConfigurator.setSimulationMode(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether this feature is enabled.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setSimulationMode(JobConf, boolean)
+ */
+ protected static Boolean getSimulationMode(JobConf job) {
+ return OutputConfigurator.getSimulationMode(CLASS, job);
+ }
+
+ /**
+ * A base class to be used to create {@link RecordWriter} instances that write to Accumulo.
+ */
+ protected static class AccumuloRecordWriter implements RecordWriter<Text,Mutation> {
+ private MultiTableBatchWriter mtbw = null;
+ private HashMap<Text,BatchWriter> bws = null;
+ private Text defaultTableName = null;
+
+ private boolean simulate = false;
+ private boolean createTables = false;
+
+ private long mutCount = 0;
+ private long valCount = 0;
+
+ private Connector conn;
+
+ protected AccumuloRecordWriter(JobConf job) throws AccumuloException, AccumuloSecurityException, IOException {
+ Level l = getLogLevel(job);
+ if (l != null)
+ log.setLevel(getLogLevel(job));
+ this.simulate = getSimulationMode(job);
+ this.createTables = canCreateTables(job);
+
+ if (simulate)
+ log.info("Simulating output only. No writes to tables will occur");
+
+ this.bws = new HashMap<Text,BatchWriter>();
+
+ String tname = getDefaultTableName(job);
+ this.defaultTableName = (tname == null) ? null : new Text(tname);
+
+ if (!simulate) {
+ this.conn = getInstance(job).getConnector(getPrincipal(job), CredentialHelper.extractToken(getTokenClass(job), getToken(job)));
+ mtbw = conn.createMultiTableBatchWriter(getBatchWriterOptions(job));
+ }
+ }
+
+ /**
+ * Push a mutation into a table. If table is null, the defaultTable will be used. If canCreateTable is set, the table will be created if it does not exist.
+ * The table name must only contain alphanumerics and underscore.
+ */
+ @Override
+ public void write(Text table, Mutation mutation) throws IOException {
+ if (table == null || table.toString().isEmpty())
+ table = this.defaultTableName;
+
+ if (!simulate && table == null)
+ throw new IOException("No table or default table specified. Try simulation mode next time");
+
+ ++mutCount;
+ valCount += mutation.size();
+ printMutation(table, mutation);
+
+ if (simulate)
+ return;
+
+ if (!bws.containsKey(table))
+ try {
+ addTable(table);
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new IOException(e);
+ }
+
+ try {
+ bws.get(table).addMutation(mutation);
+ } catch (MutationsRejectedException e) {
+ throw new IOException(e);
+ }
+ }
+
+ public void addTable(Text tableName) throws AccumuloException, AccumuloSecurityException {
+ if (simulate) {
+ log.info("Simulating adding table: " + tableName);
+ return;
+ }
+
+ log.debug("Adding table: " + tableName);
+ BatchWriter bw = null;
+ String table = tableName.toString();
+
+ if (createTables && !conn.tableOperations().exists(table)) {
+ try {
+ conn.tableOperations().create(table);
+ } catch (AccumuloSecurityException e) {
+ log.error("Accumulo security violation creating " + table, e);
+ throw e;
+ } catch (TableExistsException e) {
+ // Shouldn't happen
+ }
+ }
+
+ try {
+ bw = mtbw.getBatchWriter(table);
+ } catch (TableNotFoundException e) {
+ log.error("Accumulo table " + table + " doesn't exist and cannot be created.", e);
+ throw new AccumuloException(e);
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (AccumuloSecurityException e) {
+ throw e;
+ }
+
+ if (bw != null)
+ bws.put(tableName, bw);
+ }
+
+ private int printMutation(Text table, Mutation m) {
+ if (log.isTraceEnabled()) {
+ log.trace(String.format("Table %s row key: %s", table, hexDump(m.getRow())));
+ for (ColumnUpdate cu : m.getUpdates()) {
+ log.trace(String.format("Table %s column: %s:%s", table, hexDump(cu.getColumnFamily()), hexDump(cu.getColumnQualifier())));
+ log.trace(String.format("Table %s security: %s", table, new ColumnVisibility(cu.getColumnVisibility()).toString()));
+ log.trace(String.format("Table %s value: %s", table, hexDump(cu.getValue())));
+ }
+ }
+ return m.getUpdates().size();
+ }
+
+ private String hexDump(byte[] ba) {
+ StringBuilder sb = new StringBuilder();
+ for (byte b : ba) {
+ if ((b > 0x20) && (b < 0x7e))
+ sb.append((char) b);
+ else
+ sb.append(String.format("x%02x", b));
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public void close(Reporter reporter) throws IOException {
+ log.debug("mutations written: " + mutCount + ", values written: " + valCount);
+ if (simulate)
+ return;
+
+ try {
+ mtbw.close();
+ } catch (MutationsRejectedException e) {
+ if (e.getAuthorizationFailuresMap().size() >= 0) {
+ HashMap<String,Set<SecurityErrorCode>> tables = new HashMap<String,Set<SecurityErrorCode>>();
+ for (Entry<KeyExtent,Set<SecurityErrorCode>> ke : e.getAuthorizationFailuresMap().entrySet()) {
+ Set<SecurityErrorCode> secCodes = tables.get(ke.getKey().getTableId().toString());
+ if (secCodes == null) {
+ secCodes = new HashSet<SecurityErrorCode>();
+ tables.put(ke.getKey().getTableId().toString(), secCodes);
+ }
+ secCodes.addAll(ke.getValue());
+ }
+
+ log.error("Not authorized to write to tables : " + tables);
+ }
+
+ if (e.getConstraintViolationSummaries().size() > 0) {
+ log.error("Constraint violations : " + e.getConstraintViolationSummaries().size());
+ }
+ }
+ }
+ }
+
+ @Override
+ public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException {
+ if (!isConnectorInfoSet(job))
+ throw new IOException("Connector info has not been set.");
+ try {
+ // if the instance isn't configured, it will complain here
+ Connector c = getInstance(job).getConnector(getPrincipal(job), CredentialHelper.extractToken(getTokenClass(job), getToken(job)));
+ if (!c.securityOperations().authenticateUser(getPrincipal(job), CredentialHelper.extractToken(getTokenClass(job), getToken(job))))
+ throw new IOException("Unable to authenticate user");
+ } catch (AccumuloException e) {
+ throw new IOException(e);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public RecordWriter<Text,Mutation> getRecordWriter(FileSystem ignored, JobConf job, String name, Progressable progress) throws IOException {
+ try {
+ return new AccumuloRecordWriter(job);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
index 16efa89,0000000..bc568e8
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
@@@ -1,925 -1,0 +1,924 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapred;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientSideIteratorScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.RowIterator;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableDeletedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.impl.OfflineScanner;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.TabletLocator;
+import org.apache.accumulo.core.client.mapreduce.lib.util.InputConfigurator;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * This abstract {@link InputFormat} class allows MapReduce jobs to use Accumulo as the source of K,V pairs.
+ * <p>
+ * Subclasses must implement a {@link #getRecordReader(InputSplit, JobConf, Reporter)} to provide a {@link RecordReader} for K,V.
+ * <p>
+ * A static base class, RecordReaderBase, is provided to retrieve Accumulo {@link Key}/{@link Value} pairs, but one must implement its
+ * {@link RecordReaderBase#next(Object, Object)} to transform them to the desired generic types K,V.
+ * <p>
+ * See {@link AccumuloInputFormat} for an example implementation.
+ */
+public abstract class InputFormatBase<K,V> implements InputFormat<K,V> {
+
+ private static final Class<?> CLASS = AccumuloInputFormat.class;
+ protected static final Logger log = Logger.getLogger(CLASS);
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
+ * conversion to a string, and is not intended to be secure.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission)
+ * @param token
+ * the user's password
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public static void setConnectorInfo(JobConf job, String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ InputConfigurator.setConnectorInfo(CLASS, job, principal, token);
+ }
+
+ /**
+ * Determines if the connector has been configured.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the connector has been configured, false otherwise
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static Boolean isConnectorInfoSet(JobConf job) {
+ return InputConfigurator.isConnectorInfoSet(CLASS, job);
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static String getPrincipal(JobConf job) {
+ return InputConfigurator.getPrincipal(CLASS, job);
+ }
+
+ /**
+ * Gets the serialized token class from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static String getTokenClass(JobConf job) {
+ return InputConfigurator.getTokenClass(CLASS, job);
+ }
+
+ /**
+ * Gets the password from the configuration. WARNING: The password is stored in the Configuration and shared with all MapReduce tasks; It is BASE64 encoded to
+ * provide a charset safe conversion to a string, and is not intended to be secure.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the decoded user password
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static byte[] getToken(JobConf job) {
+ return InputConfigurator.getToken(CLASS, job);
+ }
+
+ /**
+ * Configures a {@link ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @param zooKeepers
+ * a comma-separated list of zookeeper servers
+ * @since 1.5.0
+ */
+ public static void setZooKeeperInstance(JobConf job, String instanceName, String zooKeepers) {
+ InputConfigurator.setZooKeeperInstance(CLASS, job, instanceName, zooKeepers);
+ }
+
+ /**
+ * Configures a {@link MockInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @since 1.5.0
+ */
+ public static void setMockInstance(JobConf job, String instanceName) {
+ InputConfigurator.setMockInstance(CLASS, job, instanceName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link Instance} based on the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return an Accumulo instance
+ * @since 1.5.0
+ * @see #setZooKeeperInstance(JobConf, String, String)
+ * @see #setMockInstance(JobConf, String)
+ */
+ protected static Instance getInstance(JobConf job) {
+ return InputConfigurator.getInstance(CLASS, job);
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param level
+ * the logging level
+ * @since 1.5.0
+ */
+ public static void setLogLevel(JobConf job, Level level) {
+ InputConfigurator.setLogLevel(CLASS, job, level);
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the log level
+ * @since 1.5.0
+ * @see #setLogLevel(JobConf, Level)
+ */
+ protected static Level getLogLevel(JobConf job) {
+ return InputConfigurator.getLogLevel(CLASS, job);
+ }
+
+ /**
+ * Sets the name of the input table, over which this job will scan.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.5.0
+ */
+ public static void setInputTableName(JobConf job, String tableName) {
+ InputConfigurator.setInputTableName(CLASS, job, tableName);
+ }
+
+ /**
+ * Gets the table name from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the table name
+ * @since 1.5.0
+ * @see #setInputTableName(JobConf, String)
+ */
+ protected static String getInputTableName(JobConf job) {
+ return InputConfigurator.getInputTableName(CLASS, job);
+ }
+
+ /**
+ * Sets the {@link Authorizations} used to scan. Must be a subset of the user's authorization. Defaults to the empty set.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param auths
+ * the user's authorizations
+ * @since 1.5.0
+ */
+ public static void setScanAuthorizations(JobConf job, Authorizations auths) {
+ InputConfigurator.setScanAuthorizations(CLASS, job, auths);
+ }
+
+ /**
+ * Gets the authorizations to set for the scans from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the Accumulo scan authorizations
+ * @since 1.5.0
+ * @see #setScanAuthorizations(JobConf, Authorizations)
+ */
+ protected static Authorizations getScanAuthorizations(JobConf job) {
+ return InputConfigurator.getScanAuthorizations(CLASS, job);
+ }
+
+ /**
+ * Sets the input ranges to scan for this job. If not set, the entire table will be scanned.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param ranges
+ * the ranges that will be mapped over
+ * @since 1.5.0
+ */
+ public static void setRanges(JobConf job, Collection<Range> ranges) {
+ InputConfigurator.setRanges(CLASS, job, ranges);
+ }
+
+ /**
+ * Gets the ranges to scan over from a job.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the ranges
+ * @throws IOException
+ * if the ranges have been encoded improperly
+ * @since 1.5.0
+ * @see #setRanges(JobConf, Collection)
+ */
+ protected static List<Range> getRanges(JobConf job) throws IOException {
+ return InputConfigurator.getRanges(CLASS, job);
+ }
+
+ /**
+ * Restricts the columns that will be mapped over for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param columnFamilyColumnQualifierPairs
+ * a pair of {@link Text} objects corresponding to column family and column qualifier. If the column qualifier is null, the entire column family is
+ * selected. An empty set is the default and is equivalent to scanning the all columns.
+ * @since 1.5.0
+ */
+ public static void fetchColumns(JobConf job, Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
+ InputConfigurator.fetchColumns(CLASS, job, columnFamilyColumnQualifierPairs);
+ }
+
+ /**
+ * Gets the columns to be mapped over from this job.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return a set of columns
+ * @since 1.5.0
+ * @see #fetchColumns(JobConf, Collection)
+ */
+ protected static Set<Pair<Text,Text>> getFetchedColumns(JobConf job) {
+ return InputConfigurator.getFetchedColumns(CLASS, job);
+ }
+
+ /**
+ * Encode an iterator on the input for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param cfg
+ * the configuration of the iterator
+ * @since 1.5.0
+ */
+ public static void addIterator(JobConf job, IteratorSetting cfg) {
+ InputConfigurator.addIterator(CLASS, job, cfg);
+ }
+
+ /**
+ * Gets a list of the iterator settings (for iterators to apply to a scanner) from this configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return a list of iterators
+ * @since 1.5.0
+ * @see #addIterator(JobConf, IteratorSetting)
+ */
+ protected static List<IteratorSetting> getIterators(JobConf job) {
+ return InputConfigurator.getIterators(CLASS, job);
+ }
+
+ /**
+ * Controls the automatic adjustment of ranges for this job. This feature merges overlapping ranges, then splits them to align with tablet boundaries.
+ * Disabling this feature will cause exactly one Map task to be created for each specified range. The default setting is enabled. *
+ *
+ * <p>
+ * By default, this feature is <b>enabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @see #setRanges(JobConf, Collection)
+ * @since 1.5.0
+ */
+ public static void setAutoAdjustRanges(JobConf job, boolean enableFeature) {
+ InputConfigurator.setAutoAdjustRanges(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has auto-adjust ranges enabled.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return false if the feature is disabled, true otherwise
+ * @since 1.5.0
+ * @see #setAutoAdjustRanges(JobConf, boolean)
+ */
+ protected static boolean getAutoAdjustRanges(JobConf job) {
+ return InputConfigurator.getAutoAdjustRanges(CLASS, job);
+ }
+
+ /**
+ * Controls the use of the {@link IsolatedScanner} in this job.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setScanIsolation(JobConf job, boolean enableFeature) {
+ InputConfigurator.setScanIsolation(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has isolation enabled.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setScanIsolation(JobConf, boolean)
+ */
+ protected static boolean isIsolated(JobConf job) {
+ return InputConfigurator.isIsolated(CLASS, job);
+ }
+
+ /**
+ * Controls the use of the {@link ClientSideIteratorScanner} in this job. Enabling this feature will cause the iterator stack to be constructed within the Map
+ * task, rather than within the Accumulo TServer. To use this feature, all classes needed for those iterators must be available on the classpath for the task.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setLocalIterators(JobConf job, boolean enableFeature) {
+ InputConfigurator.setLocalIterators(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration uses local iterators.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setLocalIterators(JobConf, boolean)
+ */
+ protected static boolean usesLocalIterators(JobConf job) {
+ return InputConfigurator.usesLocalIterators(CLASS, job);
+ }
+
+ /**
+ * <p>
+ * Enable reading offline tables. By default, this feature is disabled and only online tables are scanned. This will make the map reduce job directly read the
+ * table's files. If the table is not offline, then the job will fail. If the table comes online during the map reduce job, it is likely that the job will
+ * fail.
+ *
+ * <p>
+ * To use this option, the map reduce user will need access to read the Accumulo directory in HDFS.
+ *
+ * <p>
+ * Reading the offline table will create the scan time iterator stack in the map process. So any iterators that are configured for the table will need to be
+ * on the mapper's classpath.
+ *
+ * <p>
+ * One way to use this feature is to clone a table, take the clone offline, and use the clone as the input table for a map reduce job. If you plan to map
+ * reduce over the data many times, it may be better to the compact the table, clone it, take it offline, and use the clone for all map reduce jobs. The
+ * reason to do this is that compaction will reduce each tablet in the table to one file, and it is faster to read from one file.
+ *
+ * <p>
+ * There are two possible advantages to reading a tables file directly out of HDFS. First, you may see better read performance. Second, it will support
+ * speculative execution better. When reading an online table speculative execution can put more load on an already slow tablet server.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setOfflineTableScan(JobConf job, boolean enableFeature) {
+ InputConfigurator.setOfflineTableScan(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has the offline table scan feature enabled.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setOfflineTableScan(JobConf, boolean)
+ */
+ protected static boolean isOfflineScan(JobConf job) {
+ return InputConfigurator.isOfflineScan(CLASS, job);
+ }
+
+ /**
+ * Initializes an Accumulo {@link TabletLocator} based on the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return an Accumulo tablet locator
+ * @throws TableNotFoundException
+ * if the table name set on the configuration doesn't exist
+ * @since 1.5.0
+ */
+ protected static TabletLocator getTabletLocator(JobConf job) throws TableNotFoundException {
+ return InputConfigurator.getTabletLocator(CLASS, job);
+ }
+
+ // InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext job)
+ /**
+ * Check whether a configuration is fully configured to be used with an Accumulo {@link org.apache.hadoop.mapreduce.InputFormat}.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @throws IOException
+ * if the context is improperly configured
+ * @since 1.5.0
+ */
+ protected static void validateOptions(JobConf job) throws IOException {
+ InputConfigurator.validateOptions(CLASS, job);
+ }
+
+ /**
+ * An abstract base class to be used to create {@link RecordReader} instances that convert from Accumulo {@link Key}/{@link Value} pairs to the user's K/V
+ * types.
+ *
+ * Subclasses must implement {@link #next(Object, Object)} to update key and value, and also to update the following variables:
+ * <ul>
+ * <li>Key {@link #currentKey} (used for progress reporting)</li>
+ * <li>int {@link #numKeysRead} (used for progress reporting)</li>
+ * </ul>
+ */
+ protected abstract static class RecordReaderBase<K,V> implements RecordReader<K,V> {
+ protected long numKeysRead;
+ protected Iterator<Entry<Key,Value>> scannerIterator;
+ protected org.apache.accumulo.core.client.mapred.RangeInputSplit split;
+
+ /**
+ * Apply the configured iterators from the configuration to the scanner.
+ *
+ * @param scanner
+ * the scanner to configure
+ */
+ protected void setupIterators(List<IteratorSetting> iterators, Scanner scanner) {
+ for (IteratorSetting iterator : iterators) {
+ scanner.addScanIterator(iterator);
+ }
+ }
+
+ /**
+ * Initialize a scanner over the given input split using this task attempt configuration.
+ */
+ public void initialize(InputSplit inSplit, JobConf job) throws IOException {
+ Scanner scanner;
+ split = (org.apache.accumulo.core.client.mapred.RangeInputSplit) inSplit;
+ log.debug("Initializing input split: " + split.getRange());
+
+ Instance instance = split.getInstance();
+ if (null == instance) {
+ instance = getInstance(job);
+ }
+
+ String principal = split.getPrincipal();
+ if (null == principal) {
+ principal = getPrincipal(job);
+ }
+
+ AuthenticationToken token = split.getToken();
+ if (null == token) {
+ String tokenClass = getTokenClass(job);
+ byte[] tokenBytes = getToken(job);
+ try {
+ token = CredentialHelper.extractToken(tokenClass, tokenBytes);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+ }
+
+ Authorizations authorizations = split.getAuths();
+ if (null == authorizations) {
+ authorizations = getScanAuthorizations(job);
+ }
+
+ String table = split.getTable();
+ if (null == table) {
+ table = getInputTableName(job);
+ }
+
+ Boolean isOffline = split.isOffline();
+ if (null == isOffline) {
+ isOffline = isOfflineScan(job);
+ }
+
+ Boolean isIsolated = split.isIsolatedScan();
+ if (null == isIsolated) {
+ isIsolated = isIsolated(job);
+ }
+
+ Boolean usesLocalIterators = split.usesLocalIterators();
+ if (null == usesLocalIterators) {
+ usesLocalIterators = usesLocalIterators(job);
+ }
+
+ List<IteratorSetting> iterators = split.getIterators();
+ if (null == iterators) {
+ iterators = getIterators(job);
+ }
+
+ Set<Pair<Text,Text>> columns = split.getFetchedColumns();
+ if (null == columns) {
+ columns = getFetchedColumns(job);
+ }
+
+ try {
+ log.debug("Creating connector with user: " + principal);
+ Connector conn = instance.getConnector(principal, token);
+ log.debug("Creating scanner for table: " + table);
+ log.debug("Authorizations are: " + authorizations);
+ if (isOffline) {
+ String tokenClass = token.getClass().getCanonicalName();
+ ByteBuffer tokenBuffer = ByteBuffer.wrap(CredentialHelper.toBytes(token));
+ scanner = new OfflineScanner(instance, new TCredentials(principal, tokenClass, tokenBuffer, instance.getInstanceID()), Tables.getTableId(instance,
+ table), authorizations);
+ } else {
+ scanner = conn.createScanner(table, authorizations);
+ }
+ if (isIsolated) {
+ log.info("Creating isolated scanner");
+ scanner = new IsolatedScanner(scanner);
+ }
+ if (usesLocalIterators) {
+ log.info("Using local iterators");
+ scanner = new ClientSideIteratorScanner(scanner);
+ }
+ setupIterators(iterators, scanner);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ // setup a scanner within the bounds of this split
+ for (Pair<Text,Text> c : columns) {
+ if (c.getSecond() != null) {
+ log.debug("Fetching column " + c.getFirst() + ":" + c.getSecond());
+ scanner.fetchColumn(c.getFirst(), c.getSecond());
+ } else {
+ log.debug("Fetching column family " + c.getFirst());
+ scanner.fetchColumnFamily(c.getFirst());
+ }
+ }
+
+ scanner.setRange(split.getRange());
+
+ numKeysRead = 0;
+
+ // do this last after setting all scanner options
+ scannerIterator = scanner.iterator();
+ }
+
+ @Override
+ public void close() {}
+
+ @Override
+ public long getPos() throws IOException {
+ return numKeysRead;
+ }
+
+ @Override
+ public float getProgress() throws IOException {
+ if (numKeysRead > 0 && currentKey == null)
+ return 1.0f;
+ return split.getProgress(currentKey);
+ }
+
+ protected Key currentKey = null;
+
+ }
+
+ Map<String,Map<KeyExtent,List<Range>>> binOfflineTable(JobConf job, String tableName, List<Range> ranges) throws TableNotFoundException, AccumuloException,
+ AccumuloSecurityException {
+
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+
+ Instance instance = getInstance(job);
+ Connector conn = instance.getConnector(getPrincipal(job), CredentialHelper.extractToken(getTokenClass(job), getToken(job)));
+ String tableId = Tables.getTableId(instance, tableName);
+
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ Tables.clearCache(instance);
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ throw new AccumuloException("Table is online " + tableName + "(" + tableId + ") cannot scan table in offline mode ");
+ }
+ }
+
+ for (Range range : ranges) {
+ Text startRow;
+
+ if (range.getStartKey() != null)
+ startRow = range.getStartKey().getRow();
+ else
+ startRow = new Text();
+
+ Range metadataRange = new Range(new KeyExtent(new Text(tableId), startRow, null).getMetadataEntry(), true, null, false);
+ Scanner scanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ Constants.METADATA_PREV_ROW_COLUMN.fetch(scanner);
+ scanner.fetchColumnFamily(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY);
+ scanner.setRange(metadataRange);
+
+ RowIterator rowIter = new RowIterator(scanner);
+
+ KeyExtent lastExtent = null;
+
+ while (rowIter.hasNext()) {
+ Iterator<Entry<Key,Value>> row = rowIter.next();
+ String last = "";
+ KeyExtent extent = null;
+ String location = null;
+
+ while (row.hasNext()) {
+ Entry<Key,Value> entry = row.next();
+ Key key = entry.getKey();
+
+ if (key.getColumnFamily().equals(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY)) {
+ last = entry.getValue().toString();
+ }
+
+ if (key.getColumnFamily().equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY)
+ || key.getColumnFamily().equals(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY)) {
+ location = entry.getValue().toString();
+ }
+
+ if (Constants.METADATA_PREV_ROW_COLUMN.hasColumns(key)) {
+ extent = new KeyExtent(key.getRow(), entry.getValue());
+ }
+
+ }
+
+ if (location != null)
+ return null;
+
+ if (!extent.getTableId().toString().equals(tableId)) {
+ throw new AccumuloException("Saw unexpected table Id " + tableId + " " + extent);
+ }
+
+ if (lastExtent != null && !extent.isPreviousExtent(lastExtent)) {
+ throw new AccumuloException(" " + lastExtent + " is not previous extent " + extent);
+ }
+
+ Map<KeyExtent,List<Range>> tabletRanges = binnedRanges.get(last);
+ if (tabletRanges == null) {
+ tabletRanges = new HashMap<KeyExtent,List<Range>>();
+ binnedRanges.put(last, tabletRanges);
+ }
+
+ List<Range> rangeList = tabletRanges.get(extent);
+ if (rangeList == null) {
+ rangeList = new ArrayList<Range>();
+ tabletRanges.put(extent, rangeList);
+ }
+
+ rangeList.add(range);
+
+ if (extent.getEndRow() == null || range.afterEndKey(new Key(extent.getEndRow()).followingKey(PartialKey.ROW))) {
+ break;
+ }
+
+ lastExtent = extent;
+ }
+
+ }
+
+ return binnedRanges;
+ }
+
+ /**
+ * Read the metadata table to get tablets and match up ranges to them.
+ */
+ @Override
+ public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
+ Level logLevel = getLogLevel(job);
+ log.setLevel(logLevel);
+
+ validateOptions(job);
+
+ String tableName = getInputTableName(job);
+ boolean autoAdjust = getAutoAdjustRanges(job);
+ List<Range> ranges = autoAdjust ? Range.mergeOverlapping(getRanges(job)) : getRanges(job);
+ Instance instance = getInstance(job);
+ boolean offline = isOfflineScan(job);
+ boolean isolated = isIsolated(job);
+ boolean localIterators = usesLocalIterators(job);
+ boolean mockInstance = (null != instance && MockInstance.class.equals(instance.getClass()));
+ Set<Pair<Text,Text>> fetchedColumns = getFetchedColumns(job);
+ Authorizations auths = getScanAuthorizations(job);
+ String principal = getPrincipal(job);
+ String tokenClass = getTokenClass(job);
+ byte[] tokenBytes = getToken(job);
+
+ AuthenticationToken token;
+ try {
+ token = CredentialHelper.extractToken(tokenClass, tokenBytes);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+
+ List<IteratorSetting> iterators = getIterators(job);
+
+ if (ranges.isEmpty()) {
+ ranges = new ArrayList<Range>(1);
+ ranges.add(new Range());
+ }
+
+ // get the metadata information for these ranges
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+ TabletLocator tl;
+ try {
+ if (isOfflineScan(job)) {
+ binnedRanges = binOfflineTable(job, tableName, ranges);
+ while (binnedRanges == null) {
+ // Some tablets were still online, try again
+ UtilWaitThread.sleep(100 + (int) (Math.random() * 100)); // sleep randomly between 100 and 200 ms
+ binnedRanges = binOfflineTable(job, tableName, ranges);
+ }
+ } else {
+ String tableId = null;
+ tl = getTabletLocator(job);
+ // its possible that the cache could contain complete, but old information about a tables tablets... so clear it
+ tl.invalidateCache();
+ while (!tl.binRanges(ranges, binnedRanges, new TCredentials(principal, tokenClass, ByteBuffer.wrap(tokenBytes), instance.getInstanceID())).isEmpty()) {
+ if (!(instance instanceof MockInstance)) {
+ if (tableId == null)
+ tableId = Tables.getTableId(instance, tableName);
+ if (!Tables.exists(instance, tableId))
+ throw new TableDeletedException(tableId);
+ if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
+ throw new TableOfflineException(instance, tableId);
+ }
+ binnedRanges.clear();
+ log.warn("Unable to locate bins for specified ranges. Retrying.");
+ UtilWaitThread.sleep(100 + (int) (Math.random() * 100)); // sleep randomly between 100 and 200 ms
+ tl.invalidateCache();
+ }
+ }
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ ArrayList<org.apache.accumulo.core.client.mapred.RangeInputSplit> splits = new ArrayList<org.apache.accumulo.core.client.mapred.RangeInputSplit>(
+ ranges.size());
+ HashMap<Range,ArrayList<String>> splitsToAdd = null;
+
+ if (!autoAdjust)
+ splitsToAdd = new HashMap<Range,ArrayList<String>>();
+
+ HashMap<String,String> hostNameCache = new HashMap<String,String>();
+
+ for (Entry<String,Map<KeyExtent,List<Range>>> tserverBin : binnedRanges.entrySet()) {
+ String ip = tserverBin.getKey().split(":", 2)[0];
+ String location = hostNameCache.get(ip);
+ if (location == null) {
+ InetAddress inetAddress = InetAddress.getByName(ip);
+ location = inetAddress.getHostName();
+ hostNameCache.put(ip, location);
+ }
+
+ for (Entry<KeyExtent,List<Range>> extentRanges : tserverBin.getValue().entrySet()) {
+ Range ke = extentRanges.getKey().toDataRange();
+ for (Range r : extentRanges.getValue()) {
+ if (autoAdjust) {
+ // divide ranges into smaller ranges, based on the tablets
+ splits.add(new org.apache.accumulo.core.client.mapred.RangeInputSplit(ke.clip(r), new String[] {location}));
+ } else {
+ // don't divide ranges
+ ArrayList<String> locations = splitsToAdd.get(r);
+ if (locations == null)
+ locations = new ArrayList<String>(1);
+ locations.add(location);
+ splitsToAdd.put(r, locations);
+ }
+ }
+ }
+ }
+
+ if (!autoAdjust)
+ for (Entry<Range,ArrayList<String>> entry : splitsToAdd.entrySet())
+ splits.add(new org.apache.accumulo.core.client.mapred.RangeInputSplit(entry.getKey(), entry.getValue().toArray(new String[0])));
+
+ for (org.apache.accumulo.core.client.mapred.RangeInputSplit split : splits) {
+ split.setTable(tableName);
+ split.setOffline(offline);
+ split.setIsolatedScan(isolated);
+ split.setUsesLocalIterators(localIterators);
+ split.setMockInstance(mockInstance);
+ split.setFetchedColumns(fetchedColumns);
+ split.setPrincipal(principal);
+ split.setToken(token);
+ split.setInstanceName(instance.getInstanceName());
+ split.setZooKeepers(instance.getZooKeepers());
+ split.setAuths(auths);
+ split.setIterators(iterators);
+ split.setLogLevel(logLevel);
+ }
+
+ return splits.toArray(new InputSplit[splits.size()]);
+ }
+
+ /**
+ * @deprecated since 1.5.2; Use {@link org.apache.accumulo.core.client.mapred.RangeInputSplit} instead.
+ * @see org.apache.accumulo.core.client.mapred.RangeInputSplit
+ */
+ @Deprecated
+ public static class RangeInputSplit extends org.apache.accumulo.core.client.mapred.RangeInputSplit {
+ public RangeInputSplit() {
+ super();
+ }
+
+ public RangeInputSplit(Range range, String[] locations) {
+ super(range, locations);
+ }
+ }
+}
[32/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
index 448981b,0000000..442f1be
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
@@@ -1,1322 -1,0 +1,1318 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.admin;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableDeletedException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.client.impl.AccumuloServerException;
+import org.apache.accumulo.core.client.impl.ClientExec;
+import org.apache.accumulo.core.client.impl.ClientExecReturn;
+import org.apache.accumulo.core.client.impl.MasterClient;
+import org.apache.accumulo.core.client.impl.ServerClient;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.TabletLocator;
+import org.apache.accumulo.core.client.impl.TabletLocator.TabletLocation;
+import org.apache.accumulo.core.client.impl.thrift.ClientService;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.constraints.Constraint;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.master.thrift.TableOperation;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.LocalityGroupUtil;
+import org.apache.accumulo.core.util.MetadataTable;
+import org.apache.accumulo.core.util.NamingThreadFactory;
+import org.apache.accumulo.core.util.OpTimer;
+import org.apache.accumulo.core.util.StringUtil;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TApplicationException;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransportException;
+
+/**
+ * Provides a class for administering tables
+ *
+ */
+public class TableOperationsImpl extends TableOperationsHelper {
+ private Instance instance;
+ private TCredentials credentials;
+
+ public static final String CLONE_EXCLUDE_PREFIX = "!";
+
+ private static final Logger log = Logger.getLogger(TableOperations.class);
+
+ /**
+ * @param instance
+ * the connection information for this instance
+ * @param credentials
+ * the username/password for this connection
+ */
+ public TableOperationsImpl(Instance instance, TCredentials credentials) {
+ ArgumentChecker.notNull(instance, credentials);
+ this.instance = instance;
+ this.credentials = credentials;
+ }
+
+ /**
+ * Retrieve a list of tables in Accumulo.
+ *
+ * @return List of tables in accumulo
+ */
+ @Override
+ public SortedSet<String> list() {
+ OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Fetching list of tables...");
+ TreeSet<String> tableNames = new TreeSet<String>(Tables.getNameToIdMap(instance).keySet());
+ opTimer.stop("Fetched " + tableNames.size() + " table names in %DURATION%");
+ return tableNames;
+ }
+
+ /**
+ * A method to check if a table exists in Accumulo.
+ *
+ * @param tableName
+ * the name of the table
+ * @return true if the table exists
+ */
+ @Override
+ public boolean exists(String tableName) {
+ ArgumentChecker.notNull(tableName);
+ if (tableName.equals(Constants.METADATA_TABLE_NAME))
+ return true;
+
+ OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Checking if table " + tableName + "exists...");
+ boolean exists = Tables.getNameToIdMap(instance).containsKey(tableName);
+ opTimer.stop("Checked existance of " + exists + " in %DURATION%");
+ return exists;
+ }
+
+ /**
+ * Create a table with no special configuration
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableExistsException
+ * if the table already exists
+ */
+ @Override
+ public void create(String tableName) throws AccumuloException, AccumuloSecurityException, TableExistsException {
+ create(tableName, true, TimeType.MILLIS);
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param limitVersion
+ * Enables/disables the versioning iterator, which will limit the number of Key versions kept.
+ */
+ @Override
+ public void create(String tableName, boolean limitVersion) throws AccumuloException, AccumuloSecurityException, TableExistsException {
+ create(tableName, limitVersion, TimeType.MILLIS);
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param timeType
+ * specifies logical or real-time based time recording for entries in the table
+ * @param limitVersion
+ * Enables/disables the versioning iterator, which will limit the number of Key versions kept.
+ */
+ @Override
+ public void create(String tableName, boolean limitVersion, TimeType timeType) throws AccumuloException, AccumuloSecurityException, TableExistsException {
+ ArgumentChecker.notNull(tableName, timeType);
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(timeType.name().getBytes(Constants.UTF8)));
+
+ Map<String,String> opts = IteratorUtil.generateInitialTableProperties(limitVersion);
+
+ try {
+ doTableOperation(TableOperation.CREATE, args, opts);
+ } catch (TableNotFoundException e1) {
+ // should not happen
+ throw new RuntimeException(e1);
+ }
+ }
+
+ private long beginTableOperation() throws ThriftSecurityException, TException {
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ return client.beginTableOperation(Tracer.traceInfo(), credentials);
+ } catch (TTransportException tte) {
+ log.debug("Failed to call beginTableOperation(), retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ }
+
+ private void executeTableOperation(long opid, TableOperation op, List<ByteBuffer> args, Map<String,String> opts, boolean autoCleanUp)
+ throws ThriftSecurityException, TException, ThriftTableOperationException {
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ client.executeTableOperation(Tracer.traceInfo(), credentials, opid, op, args, opts, autoCleanUp);
+ break;
+ } catch (TTransportException tte) {
+ log.debug("Failed to call executeTableOperation(), retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ }
+
+ private String waitForTableOperation(long opid) throws ThriftSecurityException, TException, ThriftTableOperationException {
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ return client.waitForTableOperation(Tracer.traceInfo(), credentials, opid);
+ } catch (TTransportException tte) {
+ log.debug("Failed to call waitForTableOperation(), retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ }
+
+ private void finishTableOperation(long opid) throws ThriftSecurityException, TException {
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ client.finishTableOperation(Tracer.traceInfo(), credentials, opid);
+ break;
+ } catch (TTransportException tte) {
+ log.debug("Failed to call finishTableOperation(), retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ }
+
+ private String doTableOperation(TableOperation op, List<ByteBuffer> args, Map<String,String> opts) throws AccumuloSecurityException, TableExistsException,
+ TableNotFoundException, AccumuloException {
+ return doTableOperation(op, args, opts, true);
+ }
+
+ private String doTableOperation(TableOperation op, List<ByteBuffer> args, Map<String,String> opts, boolean wait) throws AccumuloSecurityException,
+ TableExistsException, TableNotFoundException, AccumuloException {
+ Long opid = null;
+
+ try {
+ opid = beginTableOperation();
+ executeTableOperation(opid, op, args, opts, !wait);
+ if (!wait) {
+ opid = null;
+ return null;
+ }
+ String ret = waitForTableOperation(opid);
+ Tables.clearCache(instance);
+ return ret;
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (ThriftTableOperationException e) {
+ switch (e.getType()) {
+ case EXISTS:
+ throw new TableExistsException(e);
+ case NOTFOUND:
+ throw new TableNotFoundException(e);
+ case OFFLINE:
+ throw new TableOfflineException(instance, null);
+ case OTHER:
+ default:
+ throw new AccumuloException(e.description, e);
+ }
+ } catch (Exception e) {
+ throw new AccumuloException(e.getMessage(), e);
+ } finally {
+ // always finish table op, even when exception
+ if (opid != null)
+ try {
+ finishTableOperation(opid);
+ } catch (Exception e) {
+ log.warn(e.getMessage(), e);
+ }
+ }
+ }
+
+ private static class SplitEnv {
+ private String tableName;
+ private String tableId;
+ private ExecutorService executor;
+ private CountDownLatch latch;
+ private AtomicReference<Exception> exception;
+
+ SplitEnv(String tableName, String tableId, ExecutorService executor, CountDownLatch latch, AtomicReference<Exception> exception) {
+ this.tableName = tableName;
+ this.tableId = tableId;
+ this.executor = executor;
+ this.latch = latch;
+ this.exception = exception;
+ }
+ }
+
+ private class SplitTask implements Runnable {
+
+ private List<Text> splits;
+ private SplitEnv env;
+
+ SplitTask(SplitEnv env, List<Text> splits) {
+ this.env = env;
+ this.splits = splits;
+ }
+
+ @Override
+ public void run() {
+ try {
+ if (env.exception.get() != null)
+ return;
+
+ if (splits.size() <= 2) {
+ addSplits(env.tableName, new TreeSet<Text>(splits), env.tableId);
+ for (int i = 0; i < splits.size(); i++)
+ env.latch.countDown();
+ return;
+ }
+
+ int mid = splits.size() / 2;
+
+ // split the middle split point to ensure that child task split different tablets and can therefore
+ // run in parallel
+ addSplits(env.tableName, new TreeSet<Text>(splits.subList(mid, mid + 1)), env.tableId);
+ env.latch.countDown();
+
+ env.executor.submit(new SplitTask(env, splits.subList(0, mid)));
+ env.executor.submit(new SplitTask(env, splits.subList(mid + 1, splits.size())));
+
+ } catch (Exception e) {
+ env.exception.compareAndSet(null, e);
+ }
+ }
+
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param partitionKeys
+ * a sorted set of row key values to pre-split the table on
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public void addSplits(String tableName, SortedSet<Text> partitionKeys) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ String tableId = Tables.getTableId(instance, tableName);
+
+ List<Text> splits = new ArrayList<Text>(partitionKeys);
+ // should be sorted because we copied from a sorted set, but that makes assumptions about
+ // how the copy was done so resort to be sure.
+ Collections.sort(splits);
+
+ CountDownLatch latch = new CountDownLatch(splits.size());
+ AtomicReference<Exception> exception = new AtomicReference<Exception>(null);
+
+ ExecutorService executor = Executors.newFixedThreadPool(16, new NamingThreadFactory("addSplits"));
+ try {
+ executor.submit(new SplitTask(new SplitEnv(tableName, tableId, executor, latch, exception), splits));
+
+ while (!latch.await(100, TimeUnit.MILLISECONDS)) {
+ if (exception.get() != null) {
+ executor.shutdownNow();
+ Exception excep = exception.get();
+ if (excep instanceof TableNotFoundException)
+ throw (TableNotFoundException) excep;
+ else if (excep instanceof AccumuloException)
+ throw (AccumuloException) excep;
+ else if (excep instanceof AccumuloSecurityException)
+ throw (AccumuloSecurityException) excep;
+ else if (excep instanceof RuntimeException)
+ throw (RuntimeException) excep;
+ else
+ throw new RuntimeException(excep);
+ }
+ }
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ } finally {
+ executor.shutdown();
+ }
+ }
+
+ private void addSplits(String tableName, SortedSet<Text> partitionKeys, String tableId) throws AccumuloException, AccumuloSecurityException,
+ TableNotFoundException, AccumuloServerException {
+ TabletLocator tabLocator = TabletLocator.getInstance(instance, new Text(tableId));
+
+ for (Text split : partitionKeys) {
+ boolean successful = false;
+ int attempt = 0;
+
+ while (!successful) {
+
+ if (attempt > 0)
+ UtilWaitThread.sleep(100);
+
+ attempt++;
+
+ TabletLocation tl = tabLocator.locateTablet(split, false, false, credentials);
+
+ if (tl == null) {
+ if (!Tables.exists(instance, tableId))
+ throw new TableNotFoundException(tableId, tableName, null);
+ else if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
+ throw new TableOfflineException(instance, tableId);
+ continue;
+ }
+
+ try {
+ TabletClientService.Client client = ThriftUtil.getTServerClient(tl.tablet_location, instance.getConfiguration());
+ try {
+ OpTimer opTimer = null;
+ if (log.isTraceEnabled())
+ opTimer = new OpTimer(log, Level.TRACE).start("Splitting tablet " + tl.tablet_extent + " on " + tl.tablet_location + " at " + split);
+
+ client.splitTablet(Tracer.traceInfo(), credentials, tl.tablet_extent.toThrift(), TextUtil.getByteBuffer(split));
+
+ // just split it, might as well invalidate it in the cache
+ tabLocator.invalidateCache(tl.tablet_extent);
+
+ if (opTimer != null)
+ opTimer.stop("Split tablet in %DURATION%");
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+
+ } catch (TApplicationException tae) {
+ throw new AccumuloServerException(tl.tablet_location, tae);
+ } catch (TTransportException e) {
+ tabLocator.invalidateCache(tl.tablet_location);
+ continue;
+ } catch (ThriftSecurityException e) {
+ Tables.clearCache(instance);
+ if (!Tables.exists(instance, tableId))
+ throw new TableNotFoundException(tableId, tableName, null);
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (NotServingTabletException e) {
+ tabLocator.invalidateCache(tl.tablet_extent);
+ continue;
+ } catch (TException e) {
+ tabLocator.invalidateCache(tl.tablet_location);
+ continue;
+ }
+
+ successful = true;
+ }
+ }
+ }
+
+ @Override
+ public void merge(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+
+ ArgumentChecker.notNull(tableName);
+ ByteBuffer EMPTY = ByteBuffer.allocate(0);
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), start == null ? EMPTY : TextUtil.getByteBuffer(start), end == null ? EMPTY
+ : TextUtil.getByteBuffer(end));
+ Map<String,String> opts = new HashMap<String,String>();
+ try {
+ doTableOperation(TableOperation.MERGE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void deleteRows(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+
+ ArgumentChecker.notNull(tableName);
+ ByteBuffer EMPTY = ByteBuffer.allocate(0);
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), start == null ? EMPTY : TextUtil.getByteBuffer(start), end == null ? EMPTY
+ : TextUtil.getByteBuffer(end));
+ Map<String,String> opts = new HashMap<String,String>();
+ try {
+ doTableOperation(TableOperation.DELETE_RANGE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @return the split points (end-row names) for the table's current split profile
+ */
+ @Override
+ public Collection<Text> listSplits(String tableName) throws TableNotFoundException, AccumuloSecurityException {
+
+ ArgumentChecker.notNull(tableName);
+
+ String tableId = Tables.getTableId(instance, tableName);
+
+ SortedSet<KeyExtent> tablets = new TreeSet<KeyExtent>();
+ Map<KeyExtent,String> locations = new TreeMap<KeyExtent,String>();
+
+ while (true) {
+ try {
+ tablets.clear();
+ locations.clear();
+ // the following method throws AccumuloException for some conditions that should be retried
+ MetadataTable.getEntries(instance, credentials, tableId, true, locations, tablets);
+ break;
+ } catch (AccumuloSecurityException ase) {
+ throw ase;
+ } catch (Throwable t) {
+ if (!Tables.exists(instance, tableId)) {
+ throw new TableNotFoundException(tableId, tableName, null);
+ }
+
+ if (t instanceof RuntimeException && t.getCause() instanceof AccumuloSecurityException) {
+ throw (AccumuloSecurityException) t.getCause();
+ }
+
+ log.info(t.getMessage() + " ... retrying ...");
+ UtilWaitThread.sleep(3000);
+ }
+ }
+
+ ArrayList<Text> endRows = new ArrayList<Text>(tablets.size());
+
+ for (KeyExtent ke : tablets)
+ if (ke.getEndRow() != null)
+ endRows.add(ke.getEndRow());
+
+ return endRows;
+ }
+
+ @Deprecated
+ @Override
+ public Collection<Text> getSplits(String tableName) throws TableNotFoundException {
+ try {
+ return listSplits(tableName);
+ } catch (AccumuloSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param maxSplits
+ * specifies the maximum number of splits to return
+ * @return the split points (end-row names) for the table's current split profile, grouped into fewer splits so as not to exceed maxSplits
- * @throws TableNotFoundException
+ */
+ @Override
+ public Collection<Text> listSplits(String tableName, int maxSplits) throws TableNotFoundException, AccumuloSecurityException {
+ Collection<Text> endRows = listSplits(tableName);
+
+ if (endRows.size() <= maxSplits)
+ return endRows;
+
+ double r = (maxSplits + 1) / (double) (endRows.size());
+ double pos = 0;
+
+ ArrayList<Text> subset = new ArrayList<Text>(maxSplits);
+
+ int j = 0;
+ for (int i = 0; i < endRows.size() && j < maxSplits; i++) {
+ pos += r;
+ while (pos > 1) {
+ subset.add(((ArrayList<Text>) endRows).get(i));
+ j++;
+ pos -= 1;
+ }
+ }
+
+ return subset;
+ }
+
+ @Deprecated
+ @Override
+ public Collection<Text> getSplits(String tableName, int maxSplits) throws TableNotFoundException {
+ try {
+ return listSplits(tableName, maxSplits);
+ } catch (AccumuloSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Delete a table
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public void delete(String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)));
+ Map<String,String> opts = new HashMap<String,String>();
+
+ try {
+ doTableOperation(TableOperation.DELETE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ @Override
+ public void clone(String srcTableName, String newTableName, boolean flush, Map<String,String> propertiesToSet, Set<String> propertiesToExclude)
+ throws AccumuloSecurityException, TableNotFoundException, AccumuloException, TableExistsException {
+
+ ArgumentChecker.notNull(srcTableName, newTableName);
+
+ String srcTableId = Tables.getTableId(instance, srcTableName);
+
+ if (flush)
+ _flush(srcTableId, null, null, true);
+
+ if (propertiesToExclude == null)
+ propertiesToExclude = Collections.emptySet();
+
+ if (propertiesToSet == null)
+ propertiesToSet = Collections.emptyMap();
+
+ if (!Collections.disjoint(propertiesToExclude, propertiesToSet.keySet()))
+ throw new IllegalArgumentException("propertiesToSet and propertiesToExclude not disjoint");
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(srcTableId.getBytes(Constants.UTF8)), ByteBuffer.wrap(newTableName.getBytes(Constants.UTF8)));
+ Map<String,String> opts = new HashMap<String,String>();
+ for (Entry<String,String> entry : propertiesToSet.entrySet()) {
+ if (entry.getKey().startsWith(CLONE_EXCLUDE_PREFIX))
+ throw new IllegalArgumentException("Property can not start with " + CLONE_EXCLUDE_PREFIX);
+ opts.put(entry.getKey(), entry.getValue());
+ }
+
+ for (String prop : propertiesToExclude) {
+ opts.put(CLONE_EXCLUDE_PREFIX + prop, "");
+ }
+
+ doTableOperation(TableOperation.CLONE, args, opts);
+ }
+
+ /**
+ * Rename a table
+ *
+ * @param oldTableName
+ * the old table name
+ * @param newTableName
+ * the new table name
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the old table name does not exist
+ * @throws TableExistsException
+ * if the new table name already exists
+ */
+ @Override
+ public void rename(String oldTableName, String newTableName) throws AccumuloSecurityException, TableNotFoundException, AccumuloException,
+ TableExistsException {
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(oldTableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(newTableName.getBytes(Constants.UTF8)));
+ Map<String,String> opts = new HashMap<String,String>();
+ doTableOperation(TableOperation.RENAME, args, opts);
+ }
+
+ /**
+ * @deprecated since 1.4 {@link #flush(String, Text, Text, boolean)}
+ */
+ @Override
+ @Deprecated
+ public void flush(String tableName) throws AccumuloException, AccumuloSecurityException {
+ try {
+ flush(tableName, null, null, false);
+ } catch (TableNotFoundException e) {
+ throw new AccumuloException(e.getMessage(), e);
+ }
+ }
+
+ /**
+ * Flush a table
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
- * @throws TableNotFoundException
+ */
+ @Override
+ public void flush(String tableName, Text start, Text end, boolean wait) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+
+ String tableId = Tables.getTableId(instance, tableName);
+ _flush(tableId, start, end, wait);
+ }
+
+ @Override
+ public void compact(String tableName, Text start, Text end, boolean flush, boolean wait) throws AccumuloSecurityException, TableNotFoundException,
+ AccumuloException {
+ compact(tableName, start, end, new ArrayList<IteratorSetting>(), flush, wait);
+ }
+
+ @Override
+ public void compact(String tableName, Text start, Text end, List<IteratorSetting> iterators, boolean flush, boolean wait) throws AccumuloSecurityException,
+ TableNotFoundException, AccumuloException {
+ ArgumentChecker.notNull(tableName);
+ ByteBuffer EMPTY = ByteBuffer.allocate(0);
+
+ String tableId = Tables.getTableId(instance, tableName);
+
+ if (flush)
+ _flush(tableId, start, end, true);
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)), start == null ? EMPTY : TextUtil.getByteBuffer(start), end == null ? EMPTY
+ : TextUtil.getByteBuffer(end), ByteBuffer.wrap(IteratorUtil.encodeIteratorSettings(iterators)));
+
+ Map<String,String> opts = new HashMap<String,String>();
+ try {
+ doTableOperation(TableOperation.COMPACT, args, opts, wait);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void cancelCompaction(String tableName) throws AccumuloSecurityException, TableNotFoundException, AccumuloException {
+ String tableId = Tables.getTableId(instance, tableName);
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)));
+
+ Map<String,String> opts = new HashMap<String,String>();
+ try {
+ doTableOperation(TableOperation.COMPACT_CANCEL, args, opts, true);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ private void _flush(String tableId, Text start, Text end, boolean wait) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+
+ try {
+ long flushID;
+
+ // used to pass the table name. but the tableid associated with a table name could change between calls.
+ // so pass the tableid to both calls
+
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ flushID = client.initiateFlush(Tracer.traceInfo(), credentials, tableId);
+ break;
+ } catch (TTransportException tte) {
+ log.debug("Failed to call initiateFlush, retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ client.waitForFlush(Tracer.traceInfo(), credentials, tableId, TextUtil.getByteBuffer(start), TextUtil.getByteBuffer(end), flushID,
+ wait ? Long.MAX_VALUE : 1);
+ break;
+ } catch (TTransportException tte) {
+ log.debug("Failed to call initiateFlush, retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ } catch (ThriftSecurityException e) {
+ switch (e.getCode()) {
+ case TABLE_DOESNT_EXIST:
+ throw new TableNotFoundException(tableId, null, e.getMessage(), e);
+ default:
+ log.debug("flush security exception on table id " + tableId);
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ }
+ } catch (ThriftTableOperationException e) {
+ switch (e.getType()) {
+ case NOTFOUND:
+ throw new TableNotFoundException(e);
+ case OTHER:
+ default:
+ throw new AccumuloException(e.description, e);
+ }
+ } catch (Exception e) {
+ throw new AccumuloException(e);
+ }
+ }
+
+ /**
+ * Sets a property on a table
+ *
+ * @param tableName
+ * the name of the table
+ * @param property
+ * the name of a per-table property
+ * @param value
+ * the value to set a per-table property to
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ @Override
+ public void setProperty(final String tableName, final String property, final String value) throws AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, property, value);
+ MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() {
+ @Override
+ public void execute(MasterClientService.Client client) throws Exception {
+ client.setTableProperty(Tracer.traceInfo(), credentials, tableName, property, value);
+ }
+ });
+ }
+
+ /**
+ * Removes a property from a table
+ *
+ * @param tableName
+ * the name of the table
+ * @param property
+ * the name of a per-table property
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ @Override
+ public void removeProperty(final String tableName, final String property) throws AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, property);
+ MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() {
+ @Override
+ public void execute(MasterClientService.Client client) throws Exception {
+ client.removeTableProperty(Tracer.traceInfo(), credentials, tableName, property);
+ }
+ });
+ }
+
+ /**
+ * Gets properties of a table
+ *
+ * @param tableName
+ * the name of the table
+ * @return all properties visible by this table (system and per-table properties)
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public Iterable<Entry<String,String>> getProperties(final String tableName) throws AccumuloException, TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+ try {
+ return ServerClient.executeRaw(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() {
+ @Override
+ public Map<String,String> execute(ClientService.Client client) throws Exception {
+ return client.getTableConfiguration(Tracer.traceInfo(), credentials, tableName);
+ }
+ }).entrySet();
+ } catch (ThriftTableOperationException e) {
+ switch (e.getType()) {
+ case NOTFOUND:
+ throw new TableNotFoundException(e);
+ case OTHER:
+ default:
+ throw new AccumuloException(e.description, e);
+ }
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AccumuloException(e);
+ }
+
+ }
+
+ /**
+ * Sets a tables locality groups. A tables locality groups can be changed at any time.
+ *
+ * @param tableName
+ * the name of the table
+ * @param groups
+ * mapping of locality group names to column families in the locality group
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public void setLocalityGroups(String tableName, Map<String,Set<Text>> groups) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ // ensure locality groups do not overlap
+ HashSet<Text> all = new HashSet<Text>();
+ for (Entry<String,Set<Text>> entry : groups.entrySet()) {
+
+ if (!Collections.disjoint(all, entry.getValue())) {
+ throw new IllegalArgumentException("Group " + entry.getKey() + " overlaps with another group");
+ }
+
+ all.addAll(entry.getValue());
+ }
+
+ for (Entry<String,Set<Text>> entry : groups.entrySet()) {
+ Set<Text> colFams = entry.getValue();
+ String value = LocalityGroupUtil.encodeColumnFamilies(colFams);
+ setProperty(tableName, Property.TABLE_LOCALITY_GROUP_PREFIX + entry.getKey(), value);
+ }
+
+ setProperty(tableName, Property.TABLE_LOCALITY_GROUPS.getKey(), StringUtil.join(groups.keySet(), ","));
+
+ // remove anything extraneous
+ String prefix = Property.TABLE_LOCALITY_GROUP_PREFIX.getKey();
+ for (Entry<String,String> entry : getProperties(tableName)) {
+ String property = entry.getKey();
+ if (property.startsWith(prefix)) {
+ // this property configures a locality group, find out which
+ // one:
+ String[] parts = property.split("\\.");
+ String group = parts[parts.length - 1];
+
+ if (!groups.containsKey(group)) {
+ removeProperty(tableName, property);
+ }
+ }
+ }
+ }
+
+ /**
+ *
+ * Gets the locality groups currently set for a table.
+ *
+ * @param tableName
+ * the name of the table
+ * @return mapping of locality group names to column families in the locality group
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public Map<String,Set<Text>> getLocalityGroups(String tableName) throws AccumuloException, TableNotFoundException {
+ AccumuloConfiguration conf = new ConfigurationCopy(this.getProperties(tableName));
+ Map<String,Set<ByteSequence>> groups = LocalityGroupUtil.getLocalityGroups(conf);
+
+ Map<String,Set<Text>> groups2 = new HashMap<String,Set<Text>>();
+ for (Entry<String,Set<ByteSequence>> entry : groups.entrySet()) {
+
+ HashSet<Text> colFams = new HashSet<Text>();
+
+ for (ByteSequence bs : entry.getValue()) {
+ colFams.add(new Text(bs.toArray()));
+ }
+
+ groups2.put(entry.getKey(), colFams);
+ }
+
+ return groups2;
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param range
+ * a range to split
+ * @param maxSplits
+ * the maximum number of splits
+ * @return the range, split into smaller ranges that fall on boundaries of the table's split points as evenly as possible
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public Set<Range> splitRangeByTablets(String tableName, Range range, int maxSplits) throws AccumuloException, AccumuloSecurityException,
+ TableNotFoundException {
+ ArgumentChecker.notNull(tableName, range);
+ if (maxSplits < 1)
+ throw new IllegalArgumentException("maximum splits must be >= 1");
+ if (maxSplits == 1)
+ return Collections.singleton(range);
+
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+ String tableId = Tables.getTableId(instance, tableName);
+ TabletLocator tl = TabletLocator.getInstance(instance, new Text(tableId));
+ // its possible that the cache could contain complete, but old information about a tables tablets... so clear it
+ tl.invalidateCache();
+ while (!tl.binRanges(Collections.singletonList(range), binnedRanges, credentials).isEmpty()) {
+ if (!Tables.exists(instance, tableId))
+ throw new TableDeletedException(tableId);
+ if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
+ throw new TableOfflineException(instance, tableId);
+
+ log.warn("Unable to locate bins for specified range. Retrying.");
+ // sleep randomly between 100 and 200ms
+ UtilWaitThread.sleep(100 + (int) (Math.random() * 100));
+ binnedRanges.clear();
+ tl.invalidateCache();
+ }
+
+ // group key extents to get <= maxSplits
+ LinkedList<KeyExtent> unmergedExtents = new LinkedList<KeyExtent>();
+ List<KeyExtent> mergedExtents = new ArrayList<KeyExtent>();
+
+ for (Map<KeyExtent,List<Range>> map : binnedRanges.values())
+ unmergedExtents.addAll(map.keySet());
+
+ // the sort method is efficient for linked list
+ Collections.sort(unmergedExtents);
+
+ while (unmergedExtents.size() + mergedExtents.size() > maxSplits) {
+ if (unmergedExtents.size() >= 2) {
+ KeyExtent first = unmergedExtents.removeFirst();
+ KeyExtent second = unmergedExtents.removeFirst();
+ first.setEndRow(second.getEndRow());
+ mergedExtents.add(first);
+ } else {
+ mergedExtents.addAll(unmergedExtents);
+ unmergedExtents.clear();
+ unmergedExtents.addAll(mergedExtents);
+ mergedExtents.clear();
+ }
+
+ }
+
+ mergedExtents.addAll(unmergedExtents);
+
+ Set<Range> ranges = new HashSet<Range>();
+ for (KeyExtent k : mergedExtents)
+ ranges.add(k.toDataRange().clip(range));
+
+ return ranges;
+ }
+
+ @Override
+ public void importDirectory(String tableName, String dir, String failureDir, boolean setTime) throws IOException, AccumuloSecurityException,
+ TableNotFoundException, AccumuloException {
+ ArgumentChecker.notNull(tableName, dir, failureDir);
+ FileSystem fs = FileUtil.getFileSystem(CachedConfiguration.getInstance(), instance.getConfiguration());
+ Path dirPath = fs.makeQualified(new Path(dir));
+ Path failPath = fs.makeQualified(new Path(failureDir));
+ if (!fs.exists(dirPath))
+ throw new AccumuloException("Bulk import directory " + dir + " does not exist!");
+ if (!fs.exists(failPath))
+ throw new AccumuloException("Bulk import failure directory " + failureDir + " does not exist!");
+ FileStatus[] listStatus = fs.listStatus(failPath);
+ if (listStatus != null && listStatus.length != 0) {
+ if (listStatus.length == 1 && listStatus[0].isDir())
+ throw new AccumuloException("Bulk import directory " + failPath + " is a file");
+ throw new AccumuloException("Bulk import failure directory " + failPath + " is not empty");
+ }
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(dirPath.toString().getBytes(Constants.UTF8)),
+ ByteBuffer.wrap(failPath.toString().getBytes(Constants.UTF8)), ByteBuffer.wrap((setTime + "").getBytes(Constants.UTF8)));
+ Map<String,String> opts = new HashMap<String,String>();
+
+ try {
+ doTableOperation(TableOperation.BULK_IMPORT, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+ // return new BulkImportHelper(instance, credentials, tableName).importDirectory(new Path(dir), new Path(failureDir), numThreads, numAssignThreads,
+ // disableGC);
+ }
+
+ /**
+ *
+ * @param tableName
+ * the table to take offline
+ * @throws AccumuloException
+ * when there is a general accumulo error
+ * @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
- * @throws TableNotFoundException
+ */
+ @Override
+ public void offline(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
+
+ ArgumentChecker.notNull(tableName);
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)));
+ Map<String,String> opts = new HashMap<String,String>();
+
+ try {
+ doTableOperation(TableOperation.OFFLINE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ *
+ * @param tableName
+ * the table to take online
+ * @throws AccumuloException
+ * when there is a general accumulo error
+ * @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
- * @throws TableNotFoundException
+ */
+ @Override
+ public void online(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)));
+ Map<String,String> opts = new HashMap<String,String>();
+
+ try {
+ doTableOperation(TableOperation.ONLINE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Clears the tablet locator cache for a specified table
+ *
+ * @param tableName
+ * the name of the table
+ * @throws TableNotFoundException
+ * if table does not exist
+ */
+ @Override
+ public void clearLocatorCache(String tableName) throws TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+ TabletLocator tabLocator = TabletLocator.getInstance(instance, new Text(Tables.getTableId(instance, tableName)));
+ tabLocator.invalidateCache();
+ }
+
+ /**
+ * Get a mapping of table name to internal table id.
+ *
+ * @return the map from table name to internal table id
+ */
+ @Override
+ public Map<String,String> tableIdMap() {
+ return Tables.getNameToIdMap(instance);
+ }
+
+ @Override
+ public Text getMaxRow(String tableName, Authorizations auths, Text startRow, boolean startInclusive, Text endRow, boolean endInclusive)
+ throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, auths);
+ Scanner scanner = instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials)).createScanner(tableName, auths);
+ return FindMax.findMax(scanner, startRow, startInclusive, endRow, endInclusive);
+ }
+
+ public static Map<String,String> getExportedProps(FileSystem fs, Path path) throws IOException {
+ HashMap<String,String> props = new HashMap<String,String>();
+
+ ZipInputStream zis = new ZipInputStream(fs.open(path));
+ try {
+ ZipEntry zipEntry;
+ while ((zipEntry = zis.getNextEntry()) != null) {
+ if (zipEntry.getName().equals(Constants.EXPORT_TABLE_CONFIG_FILE)) {
+ BufferedReader in = new BufferedReader(new InputStreamReader(zis, Constants.UTF8));
+ String line;
+ while ((line = in.readLine()) != null) {
+ String sa[] = line.split("=", 2);
+ props.put(sa[0], sa[1]);
+ }
+
+ break;
+ }
+ }
+ } finally {
+ zis.close();
+ }
+ return props;
+ }
+
+ @Override
+ public void importTable(String tableName, String importDir) throws TableExistsException, AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, importDir);
+
+ try {
+ FileSystem fs = FileUtil.getFileSystem(CachedConfiguration.getInstance(), instance.getConfiguration());
+
+ Map<String,String> props = getExportedProps(fs, new Path(importDir, Constants.EXPORT_FILE));
+
+ for (Entry<String,String> prop : props.entrySet()) {
+ if (Property.isClassProperty(prop.getKey()) && !prop.getValue().contains(Constants.CORE_PACKAGE_NAME)) {
+ Logger.getLogger(this.getClass()).info(
+ "Imported table sets '" + prop.getKey() + "' to '" + prop.getValue() + "'. Ensure this class is on Accumulo classpath.");
+ }
+ }
+
+ } catch (IOException ioe) {
+ Logger.getLogger(this.getClass()).warn("Failed to check if imported table references external java classes : " + ioe.getMessage());
+ }
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(importDir.getBytes(Constants.UTF8)));
+
+ Map<String,String> opts = Collections.emptyMap();
+
+ try {
+ doTableOperation(TableOperation.IMPORT, args, opts);
+ } catch (TableNotFoundException e1) {
+ // should not happen
+ throw new RuntimeException(e1);
+ }
+
+ }
+
+ @Override
+ public void exportTable(String tableName, String exportDir) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, exportDir);
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(exportDir.getBytes(Constants.UTF8)));
+
+ Map<String,String> opts = Collections.emptyMap();
+
+ try {
+ doTableOperation(TableOperation.EXPORT, args, opts);
+ } catch (TableExistsException e1) {
+ // should not happen
+ throw new RuntimeException(e1);
+ }
+ }
+
+ @Override
+ public boolean testClassLoad(final String tableName, final String className, final String asTypeName) throws TableNotFoundException, AccumuloException,
+ AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, className, asTypeName);
+
+ try {
+ return ServerClient.executeRaw(instance, new ClientExecReturn<Boolean,ClientService.Client>() {
+ @Override
+ public Boolean execute(ClientService.Client client) throws Exception {
+ return client.checkTableClass(Tracer.traceInfo(), credentials, tableName, className, asTypeName);
+ }
+ });
+ } catch (ThriftTableOperationException e) {
+ switch (e.getType()) {
+ case NOTFOUND:
+ throw new TableNotFoundException(e);
+ case OTHER:
+ default:
+ throw new AccumuloException(e.description, e);
+ }
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AccumuloException(e);
+ }
+ }
+
+ @Override
+ public void attachIterator(String tableName, IteratorSetting setting, EnumSet<IteratorScope> scopes) throws AccumuloSecurityException, AccumuloException,
+ TableNotFoundException {
+ testClassLoad(tableName, setting.getIteratorClass(), SortedKeyValueIterator.class.getName());
+ super.attachIterator(tableName, setting, scopes);
+ }
+
+ @Override
+ public int addConstraint(String tableName, String constraintClassName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ testClassLoad(tableName, constraintClassName, Constraint.class.getName());
+ return super.addConstraint(tableName, constraintClassName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
index e00fcc8,0000000..1132e74
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
@@@ -1,414 -1,0 +1,408 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.impl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.RowIterator;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Column;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.KeyValue;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVIterator;
+import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
+import org.apache.accumulo.core.iterators.system.ColumnQualifierFilter;
+import org.apache.accumulo.core.iterators.system.DeletingIterator;
+import org.apache.accumulo.core.iterators.system.MultiIterator;
+import org.apache.accumulo.core.iterators.system.VisibilityFilter;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.LocalityGroupUtil;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.Text;
+
+class OfflineIterator implements Iterator<Entry<Key,Value>> {
+
+ static class OfflineIteratorEnvironment implements IteratorEnvironment {
+ @Override
+ public SortedKeyValueIterator<Key,Value> reserveMapFileReader(String mapFileName) throws IOException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public AccumuloConfiguration getConfig() {
+ return AccumuloConfiguration.getDefaultConfiguration();
+ }
+
+ @Override
+ public IteratorScope getIteratorScope() {
+ return IteratorScope.scan;
+ }
+
+ @Override
+ public boolean isFullMajorCompaction() {
+ return false;
+ }
+
+ private ArrayList<SortedKeyValueIterator<Key,Value>> topLevelIterators = new ArrayList<SortedKeyValueIterator<Key,Value>>();
+
+ @Override
+ public void registerSideChannel(SortedKeyValueIterator<Key,Value> iter) {
+ topLevelIterators.add(iter);
+ }
+
+ SortedKeyValueIterator<Key,Value> getTopLevelIterator(SortedKeyValueIterator<Key,Value> iter) {
+ if (topLevelIterators.isEmpty())
+ return iter;
+ ArrayList<SortedKeyValueIterator<Key,Value>> allIters = new ArrayList<SortedKeyValueIterator<Key,Value>>(topLevelIterators);
+ allIters.add(iter);
+ return new MultiIterator(allIters, false);
+ }
+ }
+
+ private SortedKeyValueIterator<Key,Value> iter;
+ private Range range;
+ private KeyExtent currentExtent;
+ private Connector conn;
+ private String tableId;
+ private Authorizations authorizations;
+ private Instance instance;
+ private ScannerOptions options;
+ private ArrayList<SortedKeyValueIterator<Key,Value>> readers;
+ private AccumuloConfiguration config;
+
- /**
- * @param instance
- * @param credentials
- * @param authorizations
- * @param table
- */
+ public OfflineIterator(ScannerOptions options, Instance instance, TCredentials credentials, Authorizations authorizations, Text table, Range range) {
+ this.options = new ScannerOptions(options);
+ this.instance = instance;
+ this.range = range;
+
+ if (this.options.fetchedColumns.size() > 0) {
+ this.range = range.bound(this.options.fetchedColumns.first(), this.options.fetchedColumns.last());
+ }
+
+ this.tableId = table.toString();
+ this.authorizations = authorizations;
+ this.readers = new ArrayList<SortedKeyValueIterator<Key,Value>>();
+
+ try {
+ conn = instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials));
+ config = new ConfigurationCopy(conn.instanceOperations().getSiteConfiguration());
+ nextTablet();
+
+ while (iter != null && !iter.hasTop())
+ nextTablet();
+
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return iter != null && iter.hasTop();
+ }
+
+ @Override
+ public Entry<Key,Value> next() {
+ try {
+ byte[] v = iter.getTopValue().get();
+ // copy just like tablet server does, do this before calling next
+ KeyValue ret = new KeyValue(new Key(iter.getTopKey()), Arrays.copyOf(v, v.length));
+
+ iter.next();
+
+ while (iter != null && !iter.hasTop())
+ nextTablet();
+
+ return ret;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * @throws TableNotFoundException
+ * @throws IOException
+ * @throws AccumuloException
+ *
+ */
+ private void nextTablet() throws TableNotFoundException, AccumuloException, IOException {
+
+ Range nextRange = null;
+
+ if (currentExtent == null) {
+ Text startRow;
+
+ if (range.getStartKey() != null)
+ startRow = range.getStartKey().getRow();
+ else
+ startRow = new Text();
+
+ nextRange = new Range(new KeyExtent(new Text(tableId), startRow, null).getMetadataEntry(), true, null, false);
+ } else {
+
+ if (currentExtent.getEndRow() == null) {
+ iter = null;
+ return;
+ }
+
+ if (range.afterEndKey(new Key(currentExtent.getEndRow()).followingKey(PartialKey.ROW))) {
+ iter = null;
+ return;
+ }
+
+ nextRange = new Range(currentExtent.getMetadataEntry(), false, null, false);
+ }
+
+ List<String> relFiles = new ArrayList<String>();
+
+ Pair<KeyExtent,String> eloc = getTabletFiles(nextRange, relFiles);
+
+ while (eloc.getSecond() != null) {
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ Tables.clearCache(instance);
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ throw new AccumuloException("Table is online " + tableId + " cannot scan tablet in offline mode " + eloc.getFirst());
+ }
+ }
+
+ UtilWaitThread.sleep(250);
+
+ eloc = getTabletFiles(nextRange, relFiles);
+ }
+
+ KeyExtent extent = eloc.getFirst();
+
+ if (!extent.getTableId().toString().equals(tableId)) {
+ throw new AccumuloException(" did not find tablets for table " + tableId + " " + extent);
+ }
+
+ if (currentExtent != null && !extent.isPreviousExtent(currentExtent))
+ throw new AccumuloException(" " + currentExtent + " is not previous extent " + extent);
+
+ String tablesDir = Constants.getTablesDir(config);
+ List<String> absFiles = new ArrayList<String>();
+ for (String relPath : relFiles) {
+ if (relPath.startsWith(".."))
+ absFiles.add(tablesDir + relPath.substring(2));
+ else
+ absFiles.add(tablesDir + "/" + tableId + relPath);
+ }
+
+ iter = createIterator(extent, absFiles);
+ iter.seek(range, LocalityGroupUtil.families(options.fetchedColumns), options.fetchedColumns.size() == 0 ? false : true);
+ currentExtent = extent;
+
+ }
+
+ private Pair<KeyExtent,String> getTabletFiles(Range nextRange, List<String> relFiles) throws TableNotFoundException {
+ Scanner scanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ scanner.setBatchSize(100);
+ scanner.setRange(nextRange);
+
+ RowIterator rowIter = new RowIterator(scanner);
+ Iterator<Entry<Key,Value>> row = rowIter.next();
+
+ KeyExtent extent = null;
+ String location = null;
+
+ while (row.hasNext()) {
+ Entry<Key,Value> entry = row.next();
+ Key key = entry.getKey();
+
+ if (key.getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
+ relFiles.add(key.getColumnQualifier().toString());
+ }
+
+ if (key.getColumnFamily().equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY)
+ || key.getColumnFamily().equals(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY)) {
+ location = entry.getValue().toString();
+ }
+
+ if (Constants.METADATA_PREV_ROW_COLUMN.hasColumns(key)) {
+ extent = new KeyExtent(key.getRow(), entry.getValue());
+ }
+
+ }
+ return new Pair<KeyExtent,String>(extent, location);
+ }
+
+ /**
+ * @param absFiles
+ * @return
+ * @throws AccumuloException
+ * @throws TableNotFoundException
+ * @throws IOException
+ */
+ private SortedKeyValueIterator<Key,Value> createIterator(KeyExtent extent, List<String> absFiles) throws TableNotFoundException, AccumuloException,
+ IOException {
+
+ // TODO share code w/ tablet - ACCUMULO-1303
+ AccumuloConfiguration acuTableConf = AccumuloConfiguration.getTableConfiguration(conn, tableId);
+
+ Configuration conf = CachedConfiguration.getInstance();
+
+ FileSystem fs = FileUtil.getFileSystem(conf, config);
+
+ for (SortedKeyValueIterator<Key,Value> reader : readers) {
+ ((FileSKVIterator) reader).close();
+ }
+
+ readers.clear();
+
+ // TODO need to close files - ACCUMULO-1303
+ for (String file : absFiles) {
+ FileSKVIterator reader = FileOperations.getInstance().openReader(file, false, fs, conf, acuTableConf, null, null);
+ readers.add(reader);
+ }
+
+ MultiIterator multiIter = new MultiIterator(readers, extent);
+
+ OfflineIteratorEnvironment iterEnv = new OfflineIteratorEnvironment();
+
+ DeletingIterator delIter = new DeletingIterator(multiIter, false);
+
+ ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(delIter);
+
+ ColumnQualifierFilter colFilter = new ColumnQualifierFilter(cfsi, new HashSet<Column>(options.fetchedColumns));
+
+ byte[] defaultSecurityLabel;
+
+ ColumnVisibility cv = new ColumnVisibility(acuTableConf.get(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY));
+ defaultSecurityLabel = cv.getExpression();
+
+ VisibilityFilter visFilter = new VisibilityFilter(colFilter, authorizations, defaultSecurityLabel);
+
+ return iterEnv.getTopLevelIterator(IteratorUtil.loadIterators(IteratorScope.scan, visFilter, extent, acuTableConf, options.serverSideIteratorList,
+ options.serverSideIteratorOptions, iterEnv, false));
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+}
+
+/**
+ *
+ */
+public class OfflineScanner extends ScannerOptions implements Scanner {
+
+ private int batchSize;
+ private int timeOut;
+ private Range range;
+
+ private Instance instance;
+ private TCredentials credentials;
+ private Authorizations authorizations;
+ private Text tableId;
+
+ public OfflineScanner(Instance instance, TCredentials credentials, String tableId, Authorizations authorizations) {
+ ArgumentChecker.notNull(instance, credentials, tableId, authorizations);
+ this.instance = instance;
+ this.credentials = credentials;
+ this.tableId = new Text(tableId);
+ this.range = new Range((Key) null, (Key) null);
+
+ this.authorizations = authorizations;
+
+ this.batchSize = Constants.SCAN_BATCH_SIZE;
+ this.timeOut = Integer.MAX_VALUE;
+ }
+
+ @Deprecated
+ @Override
+ public void setTimeOut(int timeOut) {
+ this.timeOut = timeOut;
+ }
+
+ @Deprecated
+ @Override
+ public int getTimeOut() {
+ return timeOut;
+ }
+
+ @Override
+ public void setRange(Range range) {
+ this.range = range;
+ }
+
+ @Override
+ public Range getRange() {
+ return range;
+ }
+
+ @Override
+ public void setBatchSize(int size) {
+ this.batchSize = size;
+ }
+
+ @Override
+ public int getBatchSize() {
+ return batchSize;
+ }
+
+ @Override
+ public void enableIsolation() {
+
+ }
+
+ @Override
+ public void disableIsolation() {
+
+ }
+
+ @Override
+ public Iterator<Entry<Key,Value>> iterator() {
+ return new OfflineIterator(this, instance, credentials, authorizations, tableId, range);
+ }
+
+}
[12/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
index cb54856,0000000..1a029dc
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
@@@ -1,273 -1,0 +1,272 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce.lib.util;
+
+import java.nio.charset.Charset;
+
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * @since 1.5.0
+ */
+public class ConfiguratorBase {
+
+ /**
+ * Configuration keys for {@link Instance#getConnector(String, AuthenticationToken)}.
+ *
+ * @since 1.5.0
+ */
+ public static enum ConnectorInfo {
+ IS_CONFIGURED, PRINCIPAL, TOKEN, TOKEN_CLASS
+ }
+
+ /**
+ * Configuration keys for {@link Instance}, {@link ZooKeeperInstance}, and {@link MockInstance}.
+ *
+ * @since 1.5.0
+ */
+ protected static enum InstanceOpts {
+ TYPE, NAME, ZOO_KEEPERS;
+ }
+
+ /**
+ * Configuration keys for general configuration options.
+ *
+ * @since 1.5.0
+ */
+ protected static enum GeneralOpts {
+ LOG_LEVEL
+ }
+
+ /**
+ * Provides a configuration key for a given feature enum, prefixed by the implementingClass
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param e
+ * the enum used to provide the unique part of the configuration key
+ * @return the configuration key
+ * @since 1.5.0
+ */
+ protected static String enumToConfKey(Class<?> implementingClass, Enum<?> e) {
+ return implementingClass.getSimpleName() + "." + e.getDeclaringClass().getSimpleName() + "." + StringUtils.camelize(e.name().toLowerCase());
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
+ * conversion to a string, and is not intended to be secure.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param principal
+ * a valid Accumulo user name
+ * @param token
+ * the user's password
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public static void setConnectorInfo(Class<?> implementingClass, Configuration conf, String principal, AuthenticationToken token)
+ throws AccumuloSecurityException {
+ if (isConnectorInfoSet(implementingClass, conf))
+ throw new IllegalStateException("Connector info for " + implementingClass.getSimpleName() + " can only be set once per job");
+
+ ArgumentChecker.notNull(principal, token);
+ conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL), principal);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN_CLASS), token.getClass().getCanonicalName());
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN), CredentialHelper.tokenAsBase64(token));
+ }
+
+ /**
+ * Determines if the connector info has already been set for this instance.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the connector info has already been set, false otherwise
+ * @since 1.5.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static Boolean isConnectorInfoSet(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), false);
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the principal
+ * @since 1.5.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static String getPrincipal(Class<?> implementingClass, Configuration conf) {
+ return conf.get(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL));
+ }
+
+ /**
+ * Gets the serialized token class from the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the principal
+ * @since 1.5.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static String getTokenClass(Class<?> implementingClass, Configuration conf) {
+ return conf.get(enumToConfKey(implementingClass, ConnectorInfo.TOKEN_CLASS));
+ }
+
+ /**
+ * Gets the password from the configuration. WARNING: The password is stored in the Configuration and shared with all MapReduce tasks; It is BASE64 encoded to
+ * provide a charset safe conversion to a string, and is not intended to be secure.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the decoded principal's authentication token
+ * @since 1.5.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static byte[] getToken(Class<?> implementingClass, Configuration conf) {
+ return Base64.decodeBase64(conf.get(enumToConfKey(implementingClass, ConnectorInfo.TOKEN), "").getBytes(Charset.forName("UTF-8")));
+ }
+
+ /**
+ * Configures a {@link ZooKeeperInstance} for this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param instanceName
+ * the Accumulo instance name
+ * @param zooKeepers
+ * a comma-separated list of zookeeper servers
+ * @since 1.5.0
+ */
+ public static void setZooKeeperInstance(Class<?> implementingClass, Configuration conf, String instanceName, String zooKeepers) {
+ String key = enumToConfKey(implementingClass, InstanceOpts.TYPE);
+ if (!conf.get(key, "").isEmpty())
+ throw new IllegalStateException("Instance info can only be set once per job; it has already been configured with " + conf.get(key));
+ conf.set(key, "ZooKeeperInstance");
+
+ ArgumentChecker.notNull(instanceName, zooKeepers);
+ conf.set(enumToConfKey(implementingClass, InstanceOpts.NAME), instanceName);
+ conf.set(enumToConfKey(implementingClass, InstanceOpts.ZOO_KEEPERS), zooKeepers);
+ }
+
+ /**
+ * Configures a {@link MockInstance} for this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param instanceName
+ * the Accumulo instance name
+ * @since 1.5.0
+ */
+ public static void setMockInstance(Class<?> implementingClass, Configuration conf, String instanceName) {
+ String key = enumToConfKey(implementingClass, InstanceOpts.TYPE);
+ if (!conf.get(key, "").isEmpty())
+ throw new IllegalStateException("Instance info can only be set once per job; it has already been configured with " + conf.get(key));
+ conf.set(key, "MockInstance");
+
+ ArgumentChecker.notNull(instanceName);
+ conf.set(enumToConfKey(implementingClass, InstanceOpts.NAME), instanceName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link Instance} based on the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return an Accumulo instance
+ * @since 1.5.0
+ * @see #setZooKeeperInstance(Class, Configuration, String, String)
+ * @see #setMockInstance(Class, Configuration, String)
+ */
+ public static Instance getInstance(Class<?> implementingClass, Configuration conf) {
+ String instanceType = conf.get(enumToConfKey(implementingClass, InstanceOpts.TYPE), "");
+ if ("MockInstance".equals(instanceType))
+ return new MockInstance(conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME)));
+ else if ("ZooKeeperInstance".equals(instanceType)) {
+ return new ZooKeeperInstance(conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME)), conf.get(enumToConfKey(implementingClass,
+ InstanceOpts.ZOO_KEEPERS)));
+ } else if (instanceType.isEmpty())
+ throw new IllegalStateException("Instance has not been configured for " + implementingClass.getSimpleName());
+ else
+ throw new IllegalStateException("Unrecognized instance type " + instanceType);
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param level
+ * the logging level
+ * @since 1.5.0
+ */
+ public static void setLogLevel(Class<?> implementingClass, Configuration conf, Level level) {
+ ArgumentChecker.notNull(level);
+ Logger.getLogger(implementingClass).setLevel(level);
+ conf.setInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), level.toInt());
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the log level
+ * @since 1.5.0
+ * @see #setLogLevel(Class, Configuration, Level)
+ */
+ public static Level getLogLevel(Class<?> implementingClass, Configuration conf) {
+ return Level.toLevel(conf.getInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), Level.INFO.toInt()));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/mock/MockBatchDeleter.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mock/MockBatchDeleter.java
index 67362a2,0000000..6f321ff
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockBatchDeleter.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockBatchDeleter.java
@@@ -1,77 -1,0 +1,73 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mock;
+
+import java.util.Iterator;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.BatchDeleter;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
+
+/**
+ * {@link BatchDeleter} for a {@link MockAccumulo} instance. Behaves similarly to a regular {@link BatchDeleter}, with a few exceptions:
+ * <ol>
+ * <li>There is no waiting for memory to fill before flushing</li>
+ * <li>Only one thread is used for writing</li>
+ * </ol>
+ *
+ * Otherwise, it behaves as expected.
+ */
+public class MockBatchDeleter extends MockBatchScanner implements BatchDeleter {
+
+ private final MockAccumulo acc;
+ private final String tableName;
+
+ /**
+ * Create a {@link BatchDeleter} for the specified instance on the specified table where the writer uses the specified {@link Authorizations}.
- *
- * @param acc
- * @param tableName
- * @param auths
+ */
+ public MockBatchDeleter(MockAccumulo acc, String tableName, Authorizations auths) {
+ super(acc.tables.get(tableName), auths);
+ this.acc = acc;
+ this.tableName = tableName;
+ }
+
+ @Override
+ public void delete() throws MutationsRejectedException, TableNotFoundException {
+
+ BatchWriter writer = new MockBatchWriter(acc, tableName);
+ try {
+ Iterator<Entry<Key,Value>> iter = super.iterator();
+ while (iter.hasNext()) {
+ Entry<Key,Value> next = iter.next();
+ Key k = next.getKey();
+ Mutation m = new Mutation(k.getRow());
+ m.putDelete(k.getColumnFamily(), k.getColumnQualifier(), new ColumnVisibility(k.getColumnVisibility()), k.getTimestamp());
+ writer.addMutation(m);
+ }
+ } finally {
+ writer.close();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java
index 34eb3de,0000000..cb9481f
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java
@@@ -1,133 -1,0 +1,90 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mock;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.admin.ActiveCompaction;
+import org.apache.accumulo.core.client.admin.ActiveScan;
+import org.apache.accumulo.core.client.admin.InstanceOperations;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+
+/**
+ *
+ */
+public class MockInstanceOperations implements InstanceOperations {
+ MockAccumulo acu;
+
- /**
- * @param acu
- */
+ public MockInstanceOperations(MockAccumulo acu) {
+ this.acu = acu;
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#setProperty(java.lang.String, java.lang.String)
- */
+ @Override
+ public void setProperty(String property, String value) throws AccumuloException, AccumuloSecurityException {
+ acu.setProperty(property, value);
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#removeProperty(java.lang.String)
- */
+ @Override
+ public void removeProperty(String property) throws AccumuloException, AccumuloSecurityException {
+ acu.removeProperty(property);
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getSystemConfiguration()
- */
+ @Override
+ public Map<String,String> getSystemConfiguration() throws AccumuloException, AccumuloSecurityException {
+ return acu.systemProperties;
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getSiteConfiguration()
- */
+ @Override
+ public Map<String,String> getSiteConfiguration() throws AccumuloException, AccumuloSecurityException {
+ return acu.systemProperties;
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getTabletServers()
- */
+ @Override
+ public List<String> getTabletServers() {
+ return new ArrayList<String>();
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getActiveScans(java.lang.String)
- */
+ @Override
+ public List<ActiveScan> getActiveScans(String tserver) throws AccumuloException, AccumuloSecurityException {
+ return new ArrayList<ActiveScan>();
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#testClassLoad(java.lang.String, java.lang.String)
- */
+ @Override
+ public boolean testClassLoad(String className, String asTypeName) throws AccumuloException, AccumuloSecurityException {
+ try {
+ AccumuloVFSClassLoader.loadClass(className, Class.forName(asTypeName));
+ } catch (ClassNotFoundException e) {
+ e.printStackTrace();
+ return false;
+ }
+ return true;
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getActiveCompactions(java.lang.String)
- */
+ @Override
+ public List<ActiveCompaction> getActiveCompactions(String tserver) throws AccumuloException, AccumuloSecurityException {
+ return new ArrayList<ActiveCompaction>();
+ }
+
+ @Override
+ public void ping(String tserver) throws AccumuloException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/data/ColumnUpdate.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/ColumnUpdate.java
index bfba00f,0000000..78f2d15
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/data/ColumnUpdate.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/ColumnUpdate.java
@@@ -1,110 -1,0 +1,109 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
+import java.util.Arrays;
+
+/**
+ * A single column and value pair within a mutation
+ *
+ */
+
+public class ColumnUpdate {
+
+ private byte[] columnFamily;
+ private byte[] columnQualifier;
+ private byte[] columnVisibility;
+ private long timestamp;
+ private boolean hasTimestamp;
+ private byte[] val;
+ private boolean deleted;
+
+ public ColumnUpdate(byte[] cf, byte[] cq, byte[] cv, boolean hasts, long ts, boolean deleted, byte[] val) {
+ this.columnFamily = cf;
+ this.columnQualifier = cq;
+ this.columnVisibility = cv;
+ this.hasTimestamp = hasts;
+ this.timestamp = ts;
+ this.deleted = deleted;
+ this.val = val;
+ }
+
+ /**
+ * @deprecated use setTimestamp(long);
- * @param timestamp
+ */
+ @Deprecated
+ public void setSystemTimestamp(long timestamp) {
+ if (hasTimestamp)
+ throw new IllegalStateException("Cannot set system timestamp when user set a timestamp");
+ }
+
+ public boolean hasTimestamp() {
+ return hasTimestamp;
+ }
+
+ /**
+ * Returns the column
+ *
+ */
+ public byte[] getColumnFamily() {
+ return columnFamily;
+ }
+
+ public byte[] getColumnQualifier() {
+ return columnQualifier;
+ }
+
+ public byte[] getColumnVisibility() {
+ return columnVisibility;
+ }
+
+ public long getTimestamp() {
+ return this.timestamp;
+ }
+
+ public boolean isDeleted() {
+ return this.deleted;
+ }
+
+ public byte[] getValue() {
+ return this.val;
+ }
+
+ @Override
+ public String toString() {
+ return Arrays.toString(columnFamily) + ":" + Arrays.toString(columnQualifier) + " ["
+ + Arrays.toString(columnVisibility) + "] " + (hasTimestamp ? timestamp : "NO_TIME_STAMP") + " " + Arrays.toString(val) + " " + deleted;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof ColumnUpdate))
+ return false;
+ ColumnUpdate upd = (ColumnUpdate) obj;
+ return Arrays.equals(getColumnFamily(), upd.getColumnFamily()) && Arrays.equals(getColumnQualifier(), upd.getColumnQualifier())
+ && Arrays.equals(getColumnVisibility(), upd.getColumnVisibility()) && isDeleted() == upd.isDeleted() && Arrays.equals(getValue(), upd.getValue())
+ && hasTimestamp() == upd.hasTimestamp() && getTimestamp() == upd.getTimestamp();
+ }
+
+ @Override
+ public int hashCode() {
+ return Arrays.hashCode(columnFamily) + Arrays.hashCode(columnQualifier) + Arrays.hashCode(columnVisibility)
+ + (hasTimestamp ? (Boolean.TRUE.hashCode() + Long.valueOf(timestamp).hashCode()) : Boolean.FALSE.hashCode())
+ + (deleted ? Boolean.TRUE.hashCode() : (Boolean.FALSE.hashCode() + Arrays.hashCode(val)));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/data/Key.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Key.java
index 4b6867f,0000000..2b44359
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/data/Key.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Key.java
@@@ -1,863 -1,0 +1,864 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
+/**
+ * This is the Key used to store and access individual values in Accumulo. A Key is a tuple composed of a row, column family, column qualifier,
+ * column visibility, timestamp, and delete marker.
+ *
+ * Keys are comparable and therefore have a sorted order defined by {@link #compareTo(Key)}.
+ *
+ */
+
+import static org.apache.accumulo.core.util.ByteBufferUtil.toBytes;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.thrift.TKey;
+import org.apache.accumulo.core.data.thrift.TKeyValue;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.io.WritableUtils;
+
+public class Key implements WritableComparable<Key>, Cloneable {
+
+ protected byte[] row;
+ protected byte[] colFamily;
+ protected byte[] colQualifier;
+ protected byte[] colVisibility;
+ protected long timestamp;
+ protected boolean deleted;
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof Key)
+ return this.equals((Key) o, PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME_DEL);
+ return false;
+ }
+
+ private static final byte EMPTY_BYTES[] = new byte[0];
+
+ private byte[] copyIfNeeded(byte ba[], int off, int len, boolean copyData) {
+ if (len == 0)
+ return EMPTY_BYTES;
+
+ if (!copyData && ba.length == len && off == 0)
+ return ba;
+
+ byte[] copy = new byte[len];
+ System.arraycopy(ba, off, copy, 0, len);
+ return copy;
+ }
+
+ private final void init(byte r[], int rOff, int rLen, byte cf[], int cfOff, int cfLen, byte cq[], int cqOff, int cqLen, byte cv[], int cvOff, int cvLen,
+ long ts, boolean del, boolean copy) {
+ row = copyIfNeeded(r, rOff, rLen, copy);
+ colFamily = copyIfNeeded(cf, cfOff, cfLen, copy);
+ colQualifier = copyIfNeeded(cq, cqOff, cqLen, copy);
+ colVisibility = copyIfNeeded(cv, cvOff, cvLen, copy);
+ timestamp = ts;
+ deleted = del;
+ }
+
+ /**
+ * Creates a key with empty row, empty column family, empty column qualifier, empty column visibility, timestamp {@link Long#MAX_VALUE}, and delete marker
+ * false.
+ */
+ public Key() {
+ row = EMPTY_BYTES;
+ colFamily = EMPTY_BYTES;
+ colQualifier = EMPTY_BYTES;
+ colVisibility = EMPTY_BYTES;
+ timestamp = Long.MAX_VALUE;
+ deleted = false;
+ }
+
+ /**
+ * Creates a key with the specified row, empty column family, empty column qualifier, empty column visibility, timestamp {@link Long#MAX_VALUE}, and delete
+ * marker false.
+ */
+ public Key(Text row) {
+ init(row.getBytes(), 0, row.getLength(), EMPTY_BYTES, 0, 0, EMPTY_BYTES, 0, 0, EMPTY_BYTES, 0, 0, Long.MAX_VALUE, false, true);
+ }
+
+ /**
+ * Creates a key with the specified row, empty column family, empty column qualifier, empty column visibility, the specified timestamp, and delete marker
+ * false.
+ */
+ public Key(Text row, long ts) {
+ this(row);
+ timestamp = ts;
+ }
+
+ public Key(byte row[], int rOff, int rLen, byte cf[], int cfOff, int cfLen, byte cq[], int cqOff, int cqLen, byte cv[], int cvOff, int cvLen, long ts) {
+ init(row, rOff, rLen, cf, cfOff, cfLen, cq, cqOff, cqLen, cv, cvOff, cvLen, ts, false, true);
+ }
+
+ public Key(byte[] row, byte[] colFamily, byte[] colQualifier, byte[] colVisibility, long timestamp) {
+ this(row, colFamily, colQualifier, colVisibility, timestamp, false, true);
+ }
+
+ public Key(byte[] row, byte[] cf, byte[] cq, byte[] cv, long ts, boolean deleted) {
+ this(row, cf, cq, cv, ts, deleted, true);
+ }
+
+ public Key(byte[] row, byte[] cf, byte[] cq, byte[] cv, long ts, boolean deleted, boolean copy) {
+ init(row, 0, row.length, cf, 0, cf.length, cq, 0, cq.length, cv, 0, cv.length, ts, deleted, copy);
+ }
+
+ /**
+ * Creates a key with the specified row, the specified column family, empty column qualifier, empty column visibility, timestamp {@link Long#MAX_VALUE}, and
+ * delete marker false.
+ */
+ public Key(Text row, Text cf) {
+ init(row.getBytes(), 0, row.getLength(), cf.getBytes(), 0, cf.getLength(), EMPTY_BYTES, 0, 0, EMPTY_BYTES, 0, 0, Long.MAX_VALUE, false, true);
+ }
+
+ /**
+ * Creates a key with the specified row, the specified column family, the specified column qualifier, empty column visibility, timestamp
+ * {@link Long#MAX_VALUE}, and delete marker false.
+ */
+ public Key(Text row, Text cf, Text cq) {
+ init(row.getBytes(), 0, row.getLength(), cf.getBytes(), 0, cf.getLength(), cq.getBytes(), 0, cq.getLength(), EMPTY_BYTES, 0, 0, Long.MAX_VALUE, false, true);
+ }
+
+ /**
+ * Creates a key with the specified row, the specified column family, the specified column qualifier, the specified column visibility, timestamp
+ * {@link Long#MAX_VALUE}, and delete marker false.
+ */
+ public Key(Text row, Text cf, Text cq, Text cv) {
+ init(row.getBytes(), 0, row.getLength(), cf.getBytes(), 0, cf.getLength(), cq.getBytes(), 0, cq.getLength(), cv.getBytes(), 0, cv.getLength(),
+ Long.MAX_VALUE, false, true);
+ }
+
+ /**
+ * Creates a key with the specified row, the specified column family, the specified column qualifier, empty column visibility, the specified timestamp, and
+ * delete marker false.
+ */
+ public Key(Text row, Text cf, Text cq, long ts) {
+ init(row.getBytes(), 0, row.getLength(), cf.getBytes(), 0, cf.getLength(), cq.getBytes(), 0, cq.getLength(), EMPTY_BYTES, 0, 0, ts, false, true);
+ }
+
+ /**
+ * Creates a key with the specified row, the specified column family, the specified column qualifier, the specified column visibility, the specified
+ * timestamp, and delete marker false.
+ */
+ public Key(Text row, Text cf, Text cq, Text cv, long ts) {
+ init(row.getBytes(), 0, row.getLength(), cf.getBytes(), 0, cf.getLength(), cq.getBytes(), 0, cq.getLength(), cv.getBytes(), 0, cv.getLength(), ts, false,
+ true);
+ }
+
+ /**
+ * Creates a key with the specified row, the specified column family, the specified column qualifier, the specified column visibility, the specified
+ * timestamp, and delete marker false.
+ */
+ public Key(Text row, Text cf, Text cq, ColumnVisibility cv, long ts) {
+ byte[] expr = cv.getExpression();
+ init(row.getBytes(), 0, row.getLength(), cf.getBytes(), 0, cf.getLength(), cq.getBytes(), 0, cq.getLength(), expr, 0, expr.length, ts, false, true);
+ }
+
+ /**
+ * Converts CharSequence to Text and creates a Key using {@link #Key(Text)}.
+ */
+ public Key(CharSequence row) {
+ this(new Text(row.toString()));
+ }
+
+ /**
+ * Converts CharSequence to Text and creates a Key using {@link #Key(Text,Text)}.
+ */
+ public Key(CharSequence row, CharSequence cf) {
+ this(new Text(row.toString()), new Text(cf.toString()));
+ }
+
+ /**
+ * Converts CharSequence to Text and creates a Key using {@link #Key(Text,Text,Text)}.
+ */
+ public Key(CharSequence row, CharSequence cf, CharSequence cq) {
+ this(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()));
+ }
+
+ /**
+ * Converts CharSequence to Text and creates a Key using {@link #Key(Text,Text,Text,Text)}.
+ */
+ public Key(CharSequence row, CharSequence cf, CharSequence cq, CharSequence cv) {
+ this(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()), new Text(cv.toString()));
+ }
+
+ /**
+ * Converts CharSequence to Text and creates a Key using {@link #Key(Text,Text,Text,long)}.
+ */
+ public Key(CharSequence row, CharSequence cf, CharSequence cq, long ts) {
+ this(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()), ts);
+ }
+
+ /**
+ * Converts CharSequence to Text and creates a Key using {@link #Key(Text,Text,Text,Text,long)}.
+ */
+ public Key(CharSequence row, CharSequence cf, CharSequence cq, CharSequence cv, long ts) {
+ this(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()), new Text(cv.toString()), ts);
+ }
+
+ /**
+ * Converts CharSequence to Text and creates a Key using {@link #Key(Text,Text,Text,ColumnVisibility,long)}.
+ */
+ public Key(CharSequence row, CharSequence cf, CharSequence cq, ColumnVisibility cv, long ts) {
+ this(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()), new Text(cv.getExpression()), ts);
+ }
+
+ private byte[] followingArray(byte ba[]) {
+ byte[] fba = new byte[ba.length + 1];
+ System.arraycopy(ba, 0, fba, 0, ba.length);
+ fba[ba.length] = (byte) 0x00;
+ return fba;
+ }
+
+ /**
+ * Returns a key that will sort immediately after this key.
+ *
+ * @param part
+ * PartialKey except {@link PartialKey#ROW_COLFAM_COLQUAL_COLVIS_TIME_DEL}
+ */
+ public Key followingKey(PartialKey part) {
+ Key returnKey = new Key();
+ switch (part) {
+ case ROW:
+ returnKey.row = followingArray(row);
+ break;
+ case ROW_COLFAM:
+ returnKey.row = row;
+ returnKey.colFamily = followingArray(colFamily);
+ break;
+ case ROW_COLFAM_COLQUAL:
+ returnKey.row = row;
+ returnKey.colFamily = colFamily;
+ returnKey.colQualifier = followingArray(colQualifier);
+ break;
+ case ROW_COLFAM_COLQUAL_COLVIS:
+ // This isn't useful for inserting into accumulo, but may be useful for lookups.
+ returnKey.row = row;
+ returnKey.colFamily = colFamily;
+ returnKey.colQualifier = colQualifier;
+ returnKey.colVisibility = followingArray(colVisibility);
+ break;
+ case ROW_COLFAM_COLQUAL_COLVIS_TIME:
+ returnKey.row = row;
+ returnKey.colFamily = colFamily;
+ returnKey.colQualifier = colQualifier;
+ returnKey.colVisibility = colVisibility;
+ returnKey.setTimestamp(timestamp - 1);
+ returnKey.deleted = false;
+ break;
+ default:
+ throw new IllegalArgumentException("Partial key specification " + part + " disallowed");
+ }
+ return returnKey;
+ }
+
+ /**
+ * Creates a key with the same row, column family, column qualifier, column visibility, timestamp, and delete marker as the given key.
+ */
+ public Key(Key other) {
+ set(other);
+ }
+
+ public Key(TKey tkey) {
+ this.row = toBytes(tkey.row);
+ this.colFamily = toBytes(tkey.colFamily);
+ this.colQualifier = toBytes(tkey.colQualifier);
+ this.colVisibility = toBytes(tkey.colVisibility);
+ this.timestamp = tkey.timestamp;
+ this.deleted = false;
+
+ if (row == null) {
+ throw new IllegalArgumentException("null row");
+ }
+ if (colFamily == null) {
+ throw new IllegalArgumentException("null column family");
+ }
+ if (colQualifier == null) {
+ throw new IllegalArgumentException("null column qualifier");
+ }
+ if (colVisibility == null) {
+ throw new IllegalArgumentException("null column visibility");
+ }
+ }
+
+ /**
+ * This method gives users control over allocation of Text objects by copying into the passed in text.
+ *
+ * @param r
+ * the key's row will be copied into this Text
+ * @return the Text that was passed in
+ */
+
+ public Text getRow(Text r) {
+ r.set(row, 0, row.length);
+ return r;
+ }
+
+ /**
+ * This method returns a pointer to the keys internal data and does not copy it.
+ *
+ * @return ByteSequence that points to the internal key row data.
+ */
+
+ public ByteSequence getRowData() {
+ return new ArrayByteSequence(row);
+ }
+
+ /**
+ * This method allocates a Text object and copies into it.
+ *
+ * @return Text containing the row field
+ */
+
+ public Text getRow() {
+ return getRow(new Text());
+ }
+
+ /**
+ * Efficiently compare the the row of a key w/o allocating a text object and copying the row into it.
+ *
+ * @param r
+ * row to compare to keys row
+ * @return same as {@link #getRow()}.compareTo(r)
+ */
+
+ public int compareRow(Text r) {
+ return WritableComparator.compareBytes(row, 0, row.length, r.getBytes(), 0, r.getLength());
+ }
+
+ /**
+ * This method returns a pointer to the keys internal data and does not copy it.
+ *
+ * @return ByteSequence that points to the internal key column family data.
+ */
+
+ public ByteSequence getColumnFamilyData() {
+ return new ArrayByteSequence(colFamily);
+ }
+
+ /**
+ * This method gives users control over allocation of Text objects by copying into the passed in text.
+ *
+ * @param cf
+ * the key's column family will be copied into this Text
+ * @return the Text that was passed in
+ */
+
+ public Text getColumnFamily(Text cf) {
+ cf.set(colFamily, 0, colFamily.length);
+ return cf;
+ }
+
+ /**
+ * This method allocates a Text object and copies into it.
+ *
+ * @return Text containing the column family field
+ */
+
+ public Text getColumnFamily() {
+ return getColumnFamily(new Text());
+ }
+
+ /**
+ * Efficiently compare the the column family of a key w/o allocating a text object and copying the column qualifier into it.
+ *
+ * @param cf
+ * column family to compare to keys column family
+ * @return same as {@link #getColumnFamily()}.compareTo(cf)
+ */
+
+ public int compareColumnFamily(Text cf) {
+ return WritableComparator.compareBytes(colFamily, 0, colFamily.length, cf.getBytes(), 0, cf.getLength());
+ }
+
+ /**
+ * This method returns a pointer to the keys internal data and does not copy it.
+ *
+ * @return ByteSequence that points to the internal key column qualifier data.
+ */
+
+ public ByteSequence getColumnQualifierData() {
+ return new ArrayByteSequence(colQualifier);
+ }
+
+ /**
+ * This method gives users control over allocation of Text objects by copying into the passed in text.
+ *
+ * @param cq
+ * the key's column qualifier will be copied into this Text
+ * @return the Text that was passed in
+ */
+
+ public Text getColumnQualifier(Text cq) {
+ cq.set(colQualifier, 0, colQualifier.length);
+ return cq;
+ }
+
+ /**
+ * This method allocates a Text object and copies into it.
+ *
+ * @return Text containing the column qualifier field
+ */
+
+ public Text getColumnQualifier() {
+ return getColumnQualifier(new Text());
+ }
+
+ /**
+ * Efficiently compare the the column qualifier of a key w/o allocating a text object and copying the column qualifier into it.
+ *
+ * @param cq
+ * column family to compare to keys column qualifier
+ * @return same as {@link #getColumnQualifier()}.compareTo(cq)
+ */
+
+ public int compareColumnQualifier(Text cq) {
+ return WritableComparator.compareBytes(colQualifier, 0, colQualifier.length, cq.getBytes(), 0, cq.getLength());
+ }
+
+ public void setTimestamp(long ts) {
+ this.timestamp = ts;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public boolean isDeleted() {
+ return deleted;
+ }
+
+ public void setDeleted(boolean deleted) {
+ this.deleted = deleted;
+ }
+
+ /**
+ * This method returns a pointer to the keys internal data and does not copy it.
+ *
+ * @return ByteSequence that points to the internal key column visibility data.
+ */
+
+ public ByteSequence getColumnVisibilityData() {
+ return new ArrayByteSequence(colVisibility);
+ }
+
+ /**
+ * This method allocates a Text object and copies into it.
+ *
+ * @return Text containing the column visibility field
+ */
+
+ public final Text getColumnVisibility() {
+ return getColumnVisibility(new Text());
+ }
+
+ /**
+ * This method gives users control over allocation of Text objects by copying into the passed in text.
+ *
+ * @param cv
+ * the key's column visibility will be copied into this Text
+ * @return the Text that was passed in
+ */
+
+ public final Text getColumnVisibility(Text cv) {
+ cv.set(colVisibility, 0, colVisibility.length);
+ return cv;
+ }
+
+ /**
+ * This method creates a new ColumnVisibility representing the column visibility for this key
+ *
+ * WARNING: using this method may inhibit performance since a new ColumnVisibility object is created on every call.
+ *
+ * @return A new object representing the column visibility field
+ * @since 1.5.0
+ */
+ public final ColumnVisibility getColumnVisibilityParsed() {
+ return new ColumnVisibility(colVisibility);
+ }
+
+ /**
+ * Sets this key's row, column family, column qualifier, column visibility, timestamp, and delete marker to be the same as another key's.
+ */
+ public void set(Key k) {
+ row = k.row;
+ colFamily = k.colFamily;
+ colQualifier = k.colQualifier;
+ colVisibility = k.colVisibility;
+ timestamp = k.timestamp;
+ deleted = k.deleted;
+
+ }
+
++ @Override
+ public void readFields(DataInput in) throws IOException {
+ // this method is a little screwy so it will be compatible with older
+ // code that serialized data
+
+ int colFamilyOffset = WritableUtils.readVInt(in);
+ int colQualifierOffset = WritableUtils.readVInt(in);
+ int colVisibilityOffset = WritableUtils.readVInt(in);
+ int totalLen = WritableUtils.readVInt(in);
+
+ row = new byte[colFamilyOffset];
+ colFamily = new byte[colQualifierOffset - colFamilyOffset];
+ colQualifier = new byte[colVisibilityOffset - colQualifierOffset];
+ colVisibility = new byte[totalLen - colVisibilityOffset];
+
+ in.readFully(row);
+ in.readFully(colFamily);
+ in.readFully(colQualifier);
+ in.readFully(colVisibility);
+
+ timestamp = WritableUtils.readVLong(in);
+ deleted = in.readBoolean();
+ }
+
++ @Override
+ public void write(DataOutput out) throws IOException {
+
+ int colFamilyOffset = row.length;
+ int colQualifierOffset = colFamilyOffset + colFamily.length;
+ int colVisibilityOffset = colQualifierOffset + colQualifier.length;
+ int totalLen = colVisibilityOffset + colVisibility.length;
+
+ WritableUtils.writeVInt(out, colFamilyOffset);
+ WritableUtils.writeVInt(out, colQualifierOffset);
+ WritableUtils.writeVInt(out, colVisibilityOffset);
+
+ WritableUtils.writeVInt(out, totalLen);
+
+ out.write(row);
+ out.write(colFamily);
+ out.write(colQualifier);
+ out.write(colVisibility);
+
+ WritableUtils.writeVLong(out, timestamp);
+ out.writeBoolean(deleted);
+ }
+
+ /**
+ * Compare part of a key. For example compare just the row and column family, and if those are equal then return true.
+ *
+ */
+
+ public boolean equals(Key other, PartialKey part) {
+ switch (part) {
+ case ROW:
+ return isEqual(row, other.row);
+ case ROW_COLFAM:
+ return isEqual(row, other.row) && isEqual(colFamily, other.colFamily);
+ case ROW_COLFAM_COLQUAL:
+ return isEqual(row, other.row) && isEqual(colFamily, other.colFamily) && isEqual(colQualifier, other.colQualifier);
+ case ROW_COLFAM_COLQUAL_COLVIS:
+ return isEqual(row, other.row) && isEqual(colFamily, other.colFamily) && isEqual(colQualifier, other.colQualifier)
+ && isEqual(colVisibility, other.colVisibility);
+ case ROW_COLFAM_COLQUAL_COLVIS_TIME:
+ return isEqual(row, other.row) && isEqual(colFamily, other.colFamily) && isEqual(colQualifier, other.colQualifier)
+ && isEqual(colVisibility, other.colVisibility) && timestamp == other.timestamp;
+ case ROW_COLFAM_COLQUAL_COLVIS_TIME_DEL:
+ return isEqual(row, other.row) && isEqual(colFamily, other.colFamily) && isEqual(colQualifier, other.colQualifier)
+ && isEqual(colVisibility, other.colVisibility) && timestamp == other.timestamp && deleted == other.deleted;
+ default:
+ throw new IllegalArgumentException("Unrecognized partial key specification " + part);
+ }
+ }
+
+ /**
+ * Compare elements of a key given by a {@link PartialKey}. For example, for {@link PartialKey#ROW_COLFAM}, compare just the row and column family. If the
+ * rows are not equal, return the result of the row comparison; otherwise, return the result of the column family comparison.
+ *
+ * @see #compareTo(Key)
+ */
+
+ public int compareTo(Key other, PartialKey part) {
+ // check for matching row
+ int result = WritableComparator.compareBytes(row, 0, row.length, other.row, 0, other.row.length);
+ if (result != 0 || part.equals(PartialKey.ROW))
+ return result;
+
+ // check for matching column family
+ result = WritableComparator.compareBytes(colFamily, 0, colFamily.length, other.colFamily, 0, other.colFamily.length);
+ if (result != 0 || part.equals(PartialKey.ROW_COLFAM))
+ return result;
+
+ // check for matching column qualifier
+ result = WritableComparator.compareBytes(colQualifier, 0, colQualifier.length, other.colQualifier, 0, other.colQualifier.length);
+ if (result != 0 || part.equals(PartialKey.ROW_COLFAM_COLQUAL))
+ return result;
+
+ // check for matching column visibility
+ result = WritableComparator.compareBytes(colVisibility, 0, colVisibility.length, other.colVisibility, 0, other.colVisibility.length);
+ if (result != 0 || part.equals(PartialKey.ROW_COLFAM_COLQUAL_COLVIS))
+ return result;
+
+ // check for matching timestamp
+ if (timestamp < other.timestamp)
+ result = 1;
+ else if (timestamp > other.timestamp)
+ result = -1;
+ else
+ result = 0;
+
+ if (result != 0 || part.equals(PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME))
+ return result;
+
+ // check for matching deleted flag
+ if (deleted)
+ result = other.deleted ? 0 : -1;
+ else
+ result = other.deleted ? 1 : 0;
+
+ return result;
+ }
+
+ /**
+ * Compare all elements of a key. The elements (row, column family, column qualifier, column visibility, timestamp, and delete marker) are compared in order
+ * until an unequal element is found. If the row is equal, then compare the column family, etc. The row, column family, column qualifier, and column
+ * visibility are compared lexographically and sorted ascending. The timestamps are compared numerically and sorted descending so that the most recent data
+ * comes first. Lastly, a delete marker of true sorts before a delete marker of false.
+ */
+
++ @Override
+ public int compareTo(Key other) {
+ return compareTo(other, PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME_DEL);
+ }
+
+ @Override
+ public int hashCode() {
+ return WritableComparator.hashBytes(row, row.length) + WritableComparator.hashBytes(colFamily, colFamily.length)
+ + WritableComparator.hashBytes(colQualifier, colQualifier.length) + WritableComparator.hashBytes(colVisibility, colVisibility.length)
+ + (int) (timestamp ^ (timestamp >>> 32));
+ }
+
+ public static String toPrintableString(byte ba[], int offset, int len, int maxLen) {
+ return appendPrintableString(ba, offset, len, maxLen, new StringBuilder()).toString();
+ }
+
+ public static StringBuilder appendPrintableString(byte ba[], int offset, int len, int maxLen, StringBuilder sb) {
+ int plen = Math.min(len, maxLen);
+
+ for (int i = 0; i < plen; i++) {
+ int c = 0xff & ba[offset + i];
+ if (c >= 32 && c <= 126)
+ sb.append((char) c);
+ else
+ sb.append("%" + String.format("%02x;", c));
+ }
+
+ if (len > maxLen) {
+ sb.append("... TRUNCATED");
+ }
+
+ return sb;
+ }
+
+ private StringBuilder rowColumnStringBuilder() {
+ StringBuilder sb = new StringBuilder();
+ appendPrintableString(row, 0, row.length, Constants.MAX_DATA_TO_PRINT, sb);
+ sb.append(" ");
+ appendPrintableString(colFamily, 0, colFamily.length, Constants.MAX_DATA_TO_PRINT, sb);
+ sb.append(":");
+ appendPrintableString(colQualifier, 0, colQualifier.length, Constants.MAX_DATA_TO_PRINT, sb);
+ sb.append(" [");
+ appendPrintableString(colVisibility, 0, colVisibility.length, Constants.MAX_DATA_TO_PRINT, sb);
+ sb.append("]");
+ return sb;
+ }
+
++ @Override
+ public String toString() {
+ StringBuilder sb = rowColumnStringBuilder();
+ sb.append(" ");
+ sb.append(Long.toString(timestamp));
+ sb.append(" ");
+ sb.append(deleted);
+ return sb.toString();
+ }
+
+ public String toStringNoTime() {
+ return rowColumnStringBuilder().toString();
+ }
+
+ /**
+ * Returns the sums of the lengths of the row, column family, column qualifier, and visibility.
+ *
+ * @return row.length + colFamily.length + colQualifier.length + colVisibility.length;
+ */
+ public int getLength() {
+ return row.length + colFamily.length + colQualifier.length + colVisibility.length;
+ }
+
+ /**
+ * Same as {@link #getLength()}.
+ */
+ public int getSize() {
+ return getLength();
+ }
+
+ private static boolean isEqual(byte a1[], byte a2[]) {
+ if (a1 == a2)
+ return true;
+
+ int last = a1.length;
+
+ if (last != a2.length)
+ return false;
+
+ if (last == 0)
+ return true;
+
+ // since sorted data is usually compared in accumulo,
+ // the prefixes will normally be the same... so compare
+ // the last two charachters first.. the most likely place
+ // to have disorder is at end of the strings when the
+ // data is sorted... if those are the same compare the rest
+ // of the data forward... comparing backwards is slower
+ // (compiler and cpu optimized for reading data forward)..
+ // do not want slower comparisons when data is equal...
+ // sorting brings equals data together
+
+ last--;
+
+ if (a1[last] == a2[last]) {
+ for (int i = 0; i < last; i++)
+ if (a1[i] != a2[i])
+ return false;
+ } else {
+ return false;
+ }
+
+ return true;
+
+ }
+
+ /**
+ * Use this to compress a list of keys before sending them via thrift.
+ *
+ * @param param
+ * a list of key/value pairs
+ */
+ public static List<TKeyValue> compress(List<? extends KeyValue> param) {
+
+ List<TKeyValue> tkvl = Arrays.asList(new TKeyValue[param.size()]);
+
+ if (param.size() > 0)
+ tkvl.set(0, new TKeyValue(param.get(0).key.toThrift(), ByteBuffer.wrap(param.get(0).value)));
+
+ for (int i = param.size() - 1; i > 0; i--) {
+ Key prevKey = param.get(i - 1).key;
+ KeyValue kv = param.get(i);
+ Key key = kv.key;
+
+ TKey newKey = null;
+
+ if (isEqual(prevKey.row, key.row)) {
+ newKey = key.toThrift();
+ newKey.row = null;
+ }
+
+ if (isEqual(prevKey.colFamily, key.colFamily)) {
+ if (newKey == null)
+ newKey = key.toThrift();
+ newKey.colFamily = null;
+ }
+
+ if (isEqual(prevKey.colQualifier, key.colQualifier)) {
+ if (newKey == null)
+ newKey = key.toThrift();
+ newKey.colQualifier = null;
+ }
+
+ if (isEqual(prevKey.colVisibility, key.colVisibility)) {
+ if (newKey == null)
+ newKey = key.toThrift();
+ newKey.colVisibility = null;
+ }
+
+ if (newKey == null)
+ newKey = key.toThrift();
+
+ tkvl.set(i, new TKeyValue(newKey, ByteBuffer.wrap(kv.value)));
+ }
+
+ return tkvl;
+ }
+
+ /**
+ * Use this to decompress a list of keys received from thrift.
- *
- * @param param
+ */
-
+ public static void decompress(List<TKeyValue> param) {
+ for (int i = 1; i < param.size(); i++) {
+ TKey prevKey = param.get(i - 1).key;
+ TKey key = param.get(i).key;
+
+ if (key.row == null) {
+ key.row = prevKey.row;
+ }
+ if (key.colFamily == null) {
+ key.colFamily = prevKey.colFamily;
+ }
+ if (key.colQualifier == null) {
+ key.colQualifier = prevKey.colQualifier;
+ }
+ if (key.colVisibility == null) {
+ key.colVisibility = prevKey.colVisibility;
+ }
+ }
+ }
+
+ byte[] getRowBytes() {
+ return row;
+ }
+
+ byte[] getColFamily() {
+ return colFamily;
+ }
+
+ byte[] getColQualifier() {
+ return colQualifier;
+ }
+
+ byte[] getColVisibility() {
+ return colVisibility;
+ }
+
+ public TKey toThrift() {
+ return new TKey(ByteBuffer.wrap(row), ByteBuffer.wrap(colFamily), ByteBuffer.wrap(colQualifier), ByteBuffer.wrap(colVisibility), timestamp);
+ }
+
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ Key r = (Key) super.clone();
+ r.row = Arrays.copyOf(row, row.length);
+ r.colFamily = Arrays.copyOf(colFamily, colFamily.length);
+ r.colQualifier = Arrays.copyOf(colQualifier, colQualifier.length);
+ r.colVisibility = Arrays.copyOf(colVisibility, colVisibility.length);
+ return r;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
index e48914d,0000000..63c594c
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
@@@ -1,783 -1,0 +1,783 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
+/**
+ * keeps track of information needed to identify a tablet
+ * apparently, we only need the endKey and not the start as well
+ *
+ */
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.lang.ref.WeakReference;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.UUID;
+import java.util.WeakHashMap;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.thrift.TKeyExtent;
+import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.hadoop.io.BinaryComparable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+
+public class KeyExtent implements WritableComparable<KeyExtent> {
+
+ private static final WeakHashMap<Text,WeakReference<Text>> tableIds = new WeakHashMap<Text,WeakReference<Text>>();
+
+ private static Text dedupeTableId(Text tableId) {
+ synchronized (tableIds) {
+ WeakReference<Text> etir = tableIds.get(tableId);
+ if (etir != null) {
+ Text eti = etir.get();
+ if (eti != null) {
+ return eti;
+ }
+ }
+
+ tableId = new Text(tableId);
+ tableIds.put(tableId, new WeakReference<Text>(tableId));
+ return tableId;
+ }
+ }
+
+ private Text textTableId;
+ private Text textEndRow;
+ private Text textPrevEndRow;
+
+ private void check() {
+
+ if (getTableId() == null)
+ throw new IllegalArgumentException("null table id not allowed");
+
+ if (getEndRow() == null || getPrevEndRow() == null)
+ return;
+
+ if (getPrevEndRow().compareTo(getEndRow()) >= 0) {
+ throw new IllegalArgumentException("prevEndRow (" + getPrevEndRow() + ") >= endRow (" + getEndRow() + ")");
+ }
+ }
+
+ /**
+ * Default constructor
+ *
+ */
+ public KeyExtent() {
+ this.setTableId(new Text());
+ this.setEndRow(new Text(), false, false);
+ this.setPrevEndRow(new Text(), false, false);
+ }
+
+ public KeyExtent(Text table, Text endRow, Text prevEndRow) {
+ this.setTableId(table);
+ this.setEndRow(endRow, false, true);
+ this.setPrevEndRow(prevEndRow, false, true);
+
+ check();
+ }
+
+ public KeyExtent(KeyExtent extent) {
+ // extent has already deduped table id, so there is no need to do it again
+ this.textTableId = extent.textTableId;
+ this.setEndRow(extent.getEndRow(), false, true);
+ this.setPrevEndRow(extent.getPrevEndRow(), false, true);
+
+ check();
+ }
+
+ public KeyExtent(TKeyExtent tke) {
+ this.setTableId(new Text(ByteBufferUtil.toBytes(tke.table)));
+ this.setEndRow(tke.endRow == null ? null : new Text(ByteBufferUtil.toBytes(tke.endRow)), false, false);
+ this.setPrevEndRow(tke.prevEndRow == null ? null : new Text(ByteBufferUtil.toBytes(tke.prevEndRow)), false, false);
+
+ check();
+ }
+
+ /**
+ * Returns a String representing this extent's entry in the Metadata table
+ *
+ */
+ public Text getMetadataEntry() {
+ return getMetadataEntry(getTableId(), getEndRow());
+ }
+
+ public static Text getMetadataEntry(Text table, Text row) {
+ Text entry = new Text(table);
+
+ if (row == null) {
+ entry.append(new byte[] {'<'}, 0, 1);
+ } else {
+ entry.append(new byte[] {';'}, 0, 1);
+ entry.append(row.getBytes(), 0, row.getLength());
+ }
+
+ return entry;
+
+ }
+
+ // constructor for loading extents from metadata rows
+ public KeyExtent(Text flattenedExtent, Value prevEndRow) {
+ decodeMetadataRow(flattenedExtent);
+
+ // decode the prev row
+ this.setPrevEndRow(decodePrevEndRow(prevEndRow), false, true);
+
+ check();
+ }
+
+ // recreates an encoded extent from a string representation
+ // this encoding is what is stored as the row id of the metadata table
+ public KeyExtent(Text flattenedExtent, Text prevEndRow) {
+
+ decodeMetadataRow(flattenedExtent);
+
+ this.setPrevEndRow(null, false, false);
+ if (prevEndRow != null)
+ this.setPrevEndRow(prevEndRow, false, true);
+
+ check();
+ }
+
+ /**
+ * Sets the extents table id
+ *
+ */
+ public void setTableId(Text tId) {
+
+ if (tId == null)
+ throw new IllegalArgumentException("null table name not allowed");
+
+ this.textTableId = dedupeTableId(tId);
+
+ hashCode = 0;
+ }
+
+ /**
+ * Returns the extent's table id
+ *
+ */
+ public Text getTableId() {
+ return textTableId;
+ }
+
+ private void setEndRow(Text endRow, boolean check, boolean copy) {
+ if (endRow != null)
+ if (copy)
+ this.textEndRow = new Text(endRow);
+ else
+ this.textEndRow = endRow;
+ else
+ this.textEndRow = null;
+
+ hashCode = 0;
+ if (check)
+ check();
+ }
+
+ /**
+ * Sets this extent's end row
+ *
+ */
+ public void setEndRow(Text endRow) {
+ setEndRow(endRow, true, true);
+ }
+
+ /**
+ * Returns this extent's end row
+ *
+ */
+ public Text getEndRow() {
+ return textEndRow;
+ }
+
+ /**
+ * Return the previous extent's end row
+ *
+ */
+ public Text getPrevEndRow() {
+ return textPrevEndRow;
+ }
+
+ private void setPrevEndRow(Text prevEndRow, boolean check, boolean copy) {
+ if (prevEndRow != null)
+ if (copy)
+ this.textPrevEndRow = new Text(prevEndRow);
+ else
+ this.textPrevEndRow = prevEndRow;
+ else
+ this.textPrevEndRow = null;
+
+ hashCode = 0;
+ if (check)
+ check();
+ }
+
+ /**
+ * Sets the previous extent's end row
+ *
+ */
+ public void setPrevEndRow(Text prevEndRow) {
+ setPrevEndRow(prevEndRow, true, true);
+ }
+
+ /**
+ * Populates the extents data fields from a DataInput object
+ *
+ */
++ @Override
+ public void readFields(DataInput in) throws IOException {
+ Text tid = new Text();
+ tid.readFields(in);
+ setTableId(tid);
+ boolean hasRow = in.readBoolean();
+ if (hasRow) {
+ Text er = new Text();
+ er.readFields(in);
+ setEndRow(er, false, false);
+ } else {
+ setEndRow(null, false, false);
+ }
+ boolean hasPrevRow = in.readBoolean();
+ if (hasPrevRow) {
+ Text per = new Text();
+ per.readFields(in);
+ setPrevEndRow(per, false, true);
+ } else {
+ setPrevEndRow((Text) null);
+ }
+
+ hashCode = 0;
+ check();
+ }
+
+ /**
+ * Writes this extent's data fields to a DataOutput object
+ *
+ */
++ @Override
+ public void write(DataOutput out) throws IOException {
+ getTableId().write(out);
+ if (getEndRow() != null) {
+ out.writeBoolean(true);
+ getEndRow().write(out);
+ } else {
+ out.writeBoolean(false);
+ }
+ if (getPrevEndRow() != null) {
+ out.writeBoolean(true);
+ getPrevEndRow().write(out);
+ } else {
+ out.writeBoolean(false);
+ }
+ }
+
+ /**
+ * Returns a String representing the previous extent's entry in the Metadata table
+ *
+ */
+ public Mutation getPrevRowUpdateMutation() {
+ return getPrevRowUpdateMutation(this);
+ }
+
+ /**
+ * Empty start or end rows tell the method there are no start or end rows, and to use all the keyextents that are before the end row if no start row etc.
+ *
+ * @return all the key extents that the rows cover
+ */
+
+ public static Collection<KeyExtent> getKeyExtentsForRange(Text startRow, Text endRow, Set<KeyExtent> kes) {
+ if (kes == null)
+ return Collections.emptyList();
+ if (startRow == null)
+ startRow = new Text();
+ if (endRow == null)
+ endRow = new Text();
+ Collection<KeyExtent> keys = new ArrayList<KeyExtent>();
+ for (KeyExtent ckes : kes) {
+ if (ckes.getPrevEndRow() == null) {
+ if (ckes.getEndRow() == null) {
+ // only tablet
+ keys.add(ckes);
+ } else {
+ // first tablet
+ // if start row = '' then we want everything up to the endRow which will always include the first tablet
+ if (startRow.getLength() == 0) {
+ keys.add(ckes);
+ } else if (ckes.getEndRow().compareTo(startRow) >= 0) {
+ keys.add(ckes);
+ }
+ }
+ } else {
+ if (ckes.getEndRow() == null) {
+ // last tablet
+ // if endRow = '' and we're at the last tablet, add it
+ if (endRow.getLength() == 0) {
+ keys.add(ckes);
+ }
+ if (ckes.getPrevEndRow().compareTo(endRow) < 0) {
+ keys.add(ckes);
+ }
+ } else {
+ // tablet in the middle
+ if (startRow.getLength() == 0) {
+ // no start row
+
+ if (endRow.getLength() == 0) {
+ // no start & end row
+ keys.add(ckes);
+ } else {
+ // just no start row
+ if (ckes.getPrevEndRow().compareTo(endRow) < 0) {
+ keys.add(ckes);
+ }
+ }
+ } else if (endRow.getLength() == 0) {
+ // no end row
+ if (ckes.getEndRow().compareTo(startRow) >= 0) {
+ keys.add(ckes);
+ }
+ } else {
+ // no null prevend or endrows and no empty string start or end rows
+ if ((ckes.getPrevEndRow().compareTo(endRow) < 0 && ckes.getEndRow().compareTo(startRow) >= 0)) {
+ keys.add(ckes);
+ }
+ }
+
+ }
+ }
+ }
+ return keys;
+ }
+
+ public static Text decodePrevEndRow(Value ibw) {
+ Text per = null;
+
+ if (ibw.get()[0] != 0) {
+ per = new Text();
+ per.set(ibw.get(), 1, ibw.get().length - 1);
+ }
+
+ return per;
+ }
+
+ public static Value encodePrevEndRow(Text per) {
+ if (per == null)
+ return new Value(new byte[] {0});
+ byte[] b = new byte[per.getLength() + 1];
+ b[0] = 1;
+ System.arraycopy(per.getBytes(), 0, b, 1, per.getLength());
+ return new Value(b);
+ }
+
+ public static Mutation getPrevRowUpdateMutation(KeyExtent ke) {
+ Mutation m = new Mutation(ke.getMetadataEntry());
+ Constants.METADATA_PREV_ROW_COLUMN.put(m, encodePrevEndRow(ke.getPrevEndRow()));
+ return m;
+ }
+
+ /**
+ * Compares extents based on rows
+ *
+ */
++ @Override
+ public int compareTo(KeyExtent other) {
+
+ int result = getTableId().compareTo(other.getTableId());
+ if (result != 0)
+ return result;
+
+ if (this.getEndRow() == null) {
+ if (other.getEndRow() != null)
+ return 1;
+ } else {
+ if (other.getEndRow() == null)
+ return -1;
+
+ result = getEndRow().compareTo(other.getEndRow());
+ if (result != 0)
+ return result;
+ }
+ if (this.getPrevEndRow() == null) {
+ if (other.getPrevEndRow() == null)
+ return 0;
+ return -1;
+ }
+ if (other.getPrevEndRow() == null)
+ return 1;
+ return this.getPrevEndRow().compareTo(other.getPrevEndRow());
+ }
+
+ private int hashCode = 0;
+
+ @Override
+ public int hashCode() {
+ if (hashCode != 0)
+ return hashCode;
+
+ int prevEndRowHash = 0;
+ int endRowHash = 0;
+ if (this.getEndRow() != null) {
+ endRowHash = this.getEndRow().hashCode();
+ }
+
+ if (this.getPrevEndRow() != null) {
+ prevEndRowHash = this.getPrevEndRow().hashCode();
+ }
+
+ hashCode = getTableId().hashCode() + endRowHash + prevEndRowHash;
+ return hashCode;
+ }
+
+ private boolean equals(Text t1, Text t2) {
+ if (t1 == null || t2 == null)
+ return t1 == t2;
+
+ return t1.equals(t2);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this)
+ return true;
+ if (!(o instanceof KeyExtent))
+ return false;
+ KeyExtent oke = (KeyExtent) o;
+ return textTableId.equals(oke.textTableId) && equals(textEndRow, oke.textEndRow) && equals(textPrevEndRow, oke.textPrevEndRow);
+ }
+
+ @Override
+ public String toString() {
+ String endRowString;
+ String prevEndRowString;
+ String tableIdString = getTableId().toString().replaceAll(";", "\\\\;").replaceAll("\\\\", "\\\\\\\\");
+
+ if (getEndRow() == null)
+ endRowString = "<";
+ else
+ endRowString = ";" + TextUtil.truncate(getEndRow()).toString().replaceAll(";", "\\\\;").replaceAll("\\\\", "\\\\\\\\");
+
+ if (getPrevEndRow() == null)
+ prevEndRowString = "<";
+ else
+ prevEndRowString = ";" + TextUtil.truncate(getPrevEndRow()).toString().replaceAll(";", "\\\\;").replaceAll("\\\\", "\\\\\\\\");
+
+ return tableIdString + endRowString + prevEndRowString;
+ }
+
+ public UUID getUUID() {
+ try {
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+
+ // to get a unique hash it is important to encode the data
+ // like it is being serialized
+
+ this.write(dos);
+
+ dos.close();
+
+ return UUID.nameUUIDFromBytes(baos.toByteArray());
+
+ } catch (IOException e) {
+ // should not happen since we are writing to memory
+ throw new RuntimeException(e);
+ }
+ }
+
+ // note: this is only the encoding of the table id and the last row, not the prev row
+ /**
+ * Populates the extent's fields based on a flatted extent
+ *
+ */
+ private void decodeMetadataRow(Text flattenedExtent) {
+ int semiPos = -1;
+ int ltPos = -1;
+
+ for (int i = 0; i < flattenedExtent.getLength(); i++) {
+ if (flattenedExtent.getBytes()[i] == ';' && semiPos < 0) {
+ // want the position of the first semicolon
+ semiPos = i;
+ }
+
+ if (flattenedExtent.getBytes()[i] == '<') {
+ ltPos = i;
+ }
+ }
+
+ if (semiPos < 0 && ltPos < 0) {
+ throw new IllegalArgumentException("Metadata row does not contain ; or < " + flattenedExtent);
+ }
+
+ if (semiPos < 0) {
+
+ if (ltPos != flattenedExtent.getLength() - 1) {
+ throw new IllegalArgumentException("< must come at end of Metadata row " + flattenedExtent);
+ }
+
+ Text tableId = new Text();
+ tableId.set(flattenedExtent.getBytes(), 0, flattenedExtent.getLength() - 1);
+ this.setTableId(tableId);
+ this.setEndRow(null, false, false);
+ } else {
+
+ Text tableId = new Text();
+ tableId.set(flattenedExtent.getBytes(), 0, semiPos);
+
+ Text endRow = new Text();
+ endRow.set(flattenedExtent.getBytes(), semiPos + 1, flattenedExtent.getLength() - (semiPos + 1));
+
+ this.setTableId(tableId);
+
+ this.setEndRow(endRow, false, false);
+ }
+ }
+
+ public static byte[] tableOfMetadataRow(Text row) {
+ KeyExtent ke = new KeyExtent();
+ ke.decodeMetadataRow(row);
+ return TextUtil.getBytes(ke.getTableId());
+ }
+
+ public boolean contains(final ByteSequence bsrow) {
+ if (bsrow == null) {
+ throw new IllegalArgumentException("Passing null to contains is ambiguous, could be in first or last extent of table");
+ }
+
+ BinaryComparable row = new BinaryComparable() {
+
+ @Override
+ public int getLength() {
+ return bsrow.length();
+ }
+
+ @Override
+ public byte[] getBytes() {
+ if (bsrow.isBackedByArray() && bsrow.offset() == 0)
+ return bsrow.getBackingArray();
+
+ return bsrow.toArray();
+ }
+ };
+
+ if ((this.getPrevEndRow() == null || this.getPrevEndRow().compareTo(row) < 0) && (this.getEndRow() == null || this.getEndRow().compareTo(row) >= 0)) {
+ return true;
+ }
+ return false;
+ }
+
+ public boolean contains(BinaryComparable row) {
+ if (row == null) {
+ throw new IllegalArgumentException("Passing null to contains is ambiguous, could be in first or last extent of table");
+ }
+
+ if ((this.getPrevEndRow() == null || this.getPrevEndRow().compareTo(row) < 0) && (this.getEndRow() == null || this.getEndRow().compareTo(row) >= 0)) {
+ return true;
+ }
+ return false;
+ }
+
+ public Range toDataRange() {
+ return new Range(getPrevEndRow(), false, getEndRow(), true);
+ }
+
+ public Range toMetadataRange() {
+ Text metadataPrevRow = new Text(getTableId());
+ metadataPrevRow.append(new byte[] {';'}, 0, 1);
+ if (getPrevEndRow() != null) {
+ metadataPrevRow.append(getPrevEndRow().getBytes(), 0, getPrevEndRow().getLength());
+ }
+
+ Range range = new Range(metadataPrevRow, getPrevEndRow() == null, getMetadataEntry(), true);
+ return range;
+ }
+
+ public static SortedSet<KeyExtent> findChildren(KeyExtent ke, SortedSet<KeyExtent> tablets) {
+
+ SortedSet<KeyExtent> children = null;
+
+ for (KeyExtent tabletKe : tablets) {
+
+ if (ke.getPrevEndRow() == tabletKe.getPrevEndRow() || ke.getPrevEndRow() != null && tabletKe.getPrevEndRow() != null
+ && tabletKe.getPrevEndRow().compareTo(ke.getPrevEndRow()) == 0) {
+ children = new TreeSet<KeyExtent>();
+ }
+
+ if (children != null) {
+ children.add(tabletKe);
+ }
+
+ if (ke.getEndRow() == tabletKe.getEndRow() || ke.getEndRow() != null && tabletKe.getEndRow() != null
+ && tabletKe.getEndRow().compareTo(ke.getEndRow()) == 0) {
+ return children;
+ }
+ }
+
+ return new TreeSet<KeyExtent>();
+ }
+
+ public static KeyExtent findContainingExtent(KeyExtent extent, SortedSet<KeyExtent> extents) {
+
+ KeyExtent lookupExtent = new KeyExtent(extent);
+ lookupExtent.setPrevEndRow((Text) null);
+
+ SortedSet<KeyExtent> tailSet = extents.tailSet(lookupExtent);
+
+ if (tailSet.isEmpty()) {
+ return null;
+ }
+
+ KeyExtent first = tailSet.first();
+
+ if (first.getTableId().compareTo(extent.getTableId()) != 0) {
+ return null;
+ }
+
+ if (first.getPrevEndRow() == null) {
+ return first;
+ }
+
+ if (extent.getPrevEndRow() == null) {
+ return null;
+ }
+
+ if (extent.getPrevEndRow().compareTo(first.getPrevEndRow()) >= 0)
+ return first;
+ return null;
+ }
+
+ private static boolean startsAfter(KeyExtent nke, KeyExtent ke) {
+
+ int tiCmp = ke.getTableId().compareTo(nke.getTableId());
+
+ if (tiCmp > 0) {
+ return true;
+ }
+
+ return ke.getPrevEndRow() != null && nke.getEndRow() != null && ke.getPrevEndRow().compareTo(nke.getEndRow()) >= 0;
+ }
+
+ private static Text rowAfterPrevRow(KeyExtent nke) {
+ Text row = new Text(nke.getPrevEndRow());
+ row.append(new byte[] {0}, 0, 1);
+ return row;
+ }
+
+ // Some duplication with TabletLocatorImpl
+ public static Set<KeyExtent> findOverlapping(KeyExtent nke, SortedSet<KeyExtent> extents) {
+ if (nke == null || extents == null || extents.isEmpty())
+ return Collections.emptySet();
+
+ SortedSet<KeyExtent> start;
+
+ if (nke.getPrevEndRow() != null) {
+ Text row = rowAfterPrevRow(nke);
+ KeyExtent lookupKey = new KeyExtent(nke.getTableId(), row, null);
+ start = extents.tailSet(lookupKey);
+ } else {
+ KeyExtent lookupKey = new KeyExtent(nke.getTableId(), new Text(), null);
+ start = extents.tailSet(lookupKey);
+ }
+
+ TreeSet<KeyExtent> result = new TreeSet<KeyExtent>();
+ for (KeyExtent ke : start) {
+ if (startsAfter(nke, ke)) {
+ break;
+ }
+ result.add(ke);
+ }
+ return result;
+ }
+
+ public boolean overlaps(KeyExtent other) {
+ SortedSet<KeyExtent> set = new TreeSet<KeyExtent>();
+ set.add(other);
+ return !findOverlapping(this, set).isEmpty();
+ }
+
+ // Specialization of findOverlapping(KeyExtent, SortedSet<KeyExtent> to work with SortedMap
+ public static Set<KeyExtent> findOverlapping(KeyExtent nke, SortedMap<KeyExtent,? extends Object> extents) {
+ if (nke == null || extents == null || extents.isEmpty())
+ return Collections.emptySet();
+
+ SortedMap<KeyExtent,? extends Object> start;
+
+ if (nke.getPrevEndRow() != null) {
+ Text row = rowAfterPrevRow(nke);
+ KeyExtent lookupKey = new KeyExtent(nke.getTableId(), row, null);
+ start = extents.tailMap(lookupKey);
+ } else {
+ KeyExtent lookupKey = new KeyExtent(nke.getTableId(), new Text(), null);
+ start = extents.tailMap(lookupKey);
+ }
+
+ TreeSet<KeyExtent> result = new TreeSet<KeyExtent>();
+ for (Entry<KeyExtent,? extends Object> entry : start.entrySet()) {
+ KeyExtent ke = entry.getKey();
+ if (startsAfter(nke, ke)) {
+ break;
+ }
+ result.add(ke);
+ }
+ return result;
+ }
+
+ public static Text getMetadataEntry(KeyExtent extent) {
+ return getMetadataEntry(extent.getTableId(), extent.getEndRow());
+ }
+
+ public TKeyExtent toThrift() {
+ return new TKeyExtent(TextUtil.getByteBuffer(textTableId), textEndRow == null ? null : TextUtil.getByteBuffer(textEndRow), textPrevEndRow == null ? null
+ : TextUtil.getByteBuffer(textPrevEndRow));
+ }
+
- /**
- * @param prevExtent
- */
+ public boolean isPreviousExtent(KeyExtent prevExtent) {
+ if (prevExtent == null)
+ return getPrevEndRow() == null;
+
+ if (!prevExtent.getTableId().equals(getTableId()))
+ throw new IllegalArgumentException("Cannot compare accross tables " + prevExtent + " " + this);
+
+ if (prevExtent.getEndRow() == null)
+ return false;
+
+ if (getPrevEndRow() == null)
+ return false;
+
+ return prevExtent.getEndRow().equals(getPrevEndRow());
+ }
+
+ public boolean isMeta() {
+ return getTableId().toString().equals(Constants.METADATA_TABLE_ID);
+ }
+
+ public boolean isRootTablet() {
+ return this.compareTo(Constants.ROOT_TABLET_EXTENT) == 0;
+ }
+}
[42/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
index 8835b1c,0000000..53ea0e8
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
@@@ -1,676 -1,0 +1,672 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators.user;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.NoSuchElementException;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.OptionDescriber;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.WrappingIterator;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.VisibilityEvaluator;
+import org.apache.accumulo.core.security.VisibilityParseException;
+import org.apache.accumulo.core.util.BadArgumentException;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.commons.collections.BufferOverflowException;
+import org.apache.commons.collections.map.LRUMap;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+/**
+ * The TransformingIterator allows portions of a key (except for the row) to be transformed. This iterator handles the details that come with modifying keys
+ * (i.e., that the sort order could change). In order to do so, however, the iterator must put all keys sharing the same prefix in memory. Prefix is defined as
+ * the parts of the key that are not modified by this iterator. That is, if the iterator modifies column qualifier and timestamp, then the prefix is row and
+ * column family. In that case, the iterator must load all column qualifiers for each row/column family pair into memory. Given this constraint, care must be
+ * taken by users of this iterator to ensure it is not run in such a way that will overrun memory in a tablet server.
+ * <p>
+ * If the implementing iterator is transforming column families, then it must also override {@code untransformColumnFamilies(Collection)} to handle the case
+ * when column families are fetched at scan time. The fetched column families will/must be in the transformed space, and the untransformed column families need
+ * to be passed to this iterator's source. If it is not possible to write a reverse transformation (e.g., the column family transformation depends on the row
+ * value or something like that), then the iterator must not fetch specific column families (or only fetch column families that are known to not transform at
+ * all).
+ * <p>
+ * If the implementing iterator is transforming column visibilities, then users must be careful NOT to fetch column qualifiers from the scanner. The reason for
+ * this is due to ACCUMULO-??? (insert issue number).
+ * <p>
+ * If the implementing iterator is transforming column visibilities, then the user should be sure to supply authorizations via the {@link #AUTH_OPT} iterator
+ * option (note that this is only necessary for scan scope iterators). The supplied authorizations should be in the transformed space, but the authorizations
+ * supplied to the scanner should be in the untransformed space. That is, if the iterator transforms A to 1, B to 2, C to 3, etc, then the auths supplied when
+ * the scanner is constructed should be A,B,C,... and the auths supplied to the iterator should be 1,2,3,... The reason for this is that the scanner performs
+ * security filtering before this iterator is called, so the authorizations need to be in the original untransformed space. Since the iterator can transform
+ * visibilities, it is possible that it could produce visibilities that the user cannot see, so the transformed keys must be tested to ensure the user is
+ * allowed to view them. Note that this test is not necessary when the iterator is not used in the scan scope since no security filtering is performed during
+ * major and minor compactions. It should also be noted that this iterator implements the security filtering rather than relying on a follow-on iterator to do
+ * it so that we ensure the test is performed.
+ */
+abstract public class TransformingIterator extends WrappingIterator implements OptionDescriber {
+ public static final String AUTH_OPT = "authorizations";
+ public static final String MAX_BUFFER_SIZE_OPT = "maxBufferSize";
+ private static final long DEFAULT_MAX_BUFFER_SIZE = 10000000;
-
++
+ protected Logger log = Logger.getLogger(getClass());
-
++
+ protected ArrayList<Pair<Key,Value>> keys = new ArrayList<Pair<Key,Value>>();
+ protected int keyPos = -1;
+ protected boolean scanning;
+ protected Range seekRange;
+ protected Collection<ByteSequence> seekColumnFamilies;
+ protected boolean seekColumnFamiliesInclusive;
-
++
+ private VisibilityEvaluator ve = null;
+ private LRUMap visibleCache = null;
+ private LRUMap parsedVisibilitiesCache = null;
+ private long maxBufferSize;
-
++
+ private static Comparator<Pair<Key,Value>> keyComparator = new Comparator<Pair<Key,Value>>() {
+ @Override
+ public int compare(Pair<Key,Value> o1, Pair<Key,Value> o2) {
+ return o1.getFirst().compareTo(o2.getFirst());
+ }
+ };
-
++
+ public TransformingIterator() {}
-
++
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ super.init(source, options, env);
+ scanning = IteratorScope.scan.equals(env.getIteratorScope());
+ if (scanning) {
+ String auths = options.get(AUTH_OPT);
+ if (auths != null && !auths.isEmpty()) {
+ ve = new VisibilityEvaluator(new Authorizations(auths.getBytes(Constants.UTF8)));
+ visibleCache = new LRUMap(100);
+ }
+ }
-
++
+ if (options.containsKey(MAX_BUFFER_SIZE_OPT)) {
+ maxBufferSize = AccumuloConfiguration.getMemoryInBytes(options.get(MAX_BUFFER_SIZE_OPT));
+ } else {
+ maxBufferSize = DEFAULT_MAX_BUFFER_SIZE;
+ }
-
++
+ parsedVisibilitiesCache = new LRUMap(100);
+ }
-
++
+ @Override
+ public IteratorOptions describeOptions() {
+ String desc = "This iterator allows ranges of key to be transformed (with the exception of row transformations).";
+ String authDesc = "Comma-separated list of user's scan authorizations. "
+ + "If excluded or empty, then no visibility check is performed on transformed keys.";
+ String bufferDesc = "Maximum buffer size (in accumulo memory spec) to use for buffering keys before throwing a BufferOverflowException. "
+ + "Users should keep this limit in mind when deciding what to transform. That is, if transforming the column family for example, then all "
+ + "keys sharing the same row and column family must fit within this limit (along with their associated values)";
+ HashMap<String,String> namedOptions = new HashMap<String,String>();
+ namedOptions.put(AUTH_OPT, authDesc);
+ namedOptions.put(MAX_BUFFER_SIZE_OPT, bufferDesc);
+ return new IteratorOptions(getClass().getSimpleName(), desc, namedOptions, null);
+ }
-
++
+ @Override
+ public boolean validateOptions(Map<String,String> options) {
-
++
+ for (Entry<String,String> option : options.entrySet()) {
+ try {
+ if (option.getKey().equals(AUTH_OPT)) {
+ new Authorizations(option.getValue().getBytes(Constants.UTF8));
+ } else if (option.getKey().equals(MAX_BUFFER_SIZE_OPT)) {
+ AccumuloConfiguration.getMemoryInBytes(option.getValue());
+ }
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Failed to parse opt " + option.getKey() + " " + option.getValue(), e);
+ }
+ }
-
++
+ return true;
+ }
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ TransformingIterator copy;
-
++
+ try {
+ copy = getClass().newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
-
++
+ copy.setSource(getSource().deepCopy(env));
-
++
+ copy.scanning = scanning;
+ copy.keyPos = keyPos;
+ copy.keys.addAll(keys);
+ copy.seekRange = (seekRange == null) ? null : new Range(seekRange);
+ copy.seekColumnFamilies = (seekColumnFamilies == null) ? null : new HashSet<ByteSequence>(seekColumnFamilies);
+ copy.seekColumnFamiliesInclusive = seekColumnFamiliesInclusive;
-
++
+ copy.ve = ve;
+ if (visibleCache != null) {
+ copy.visibleCache = new LRUMap(visibleCache.maxSize());
+ copy.visibleCache.putAll(visibleCache);
+ }
-
++
+ if (parsedVisibilitiesCache != null) {
+ copy.parsedVisibilitiesCache = new LRUMap(parsedVisibilitiesCache.maxSize());
+ copy.parsedVisibilitiesCache.putAll(parsedVisibilitiesCache);
+ }
-
++
+ copy.maxBufferSize = maxBufferSize;
-
++
+ return copy;
+ }
-
++
+ @Override
+ public boolean hasTop() {
+ return keyPos >= 0 && keyPos < keys.size();
+ }
-
++
+ @Override
+ public Key getTopKey() {
+ return hasTop() ? keys.get(keyPos).getFirst() : null;
+ }
-
++
+ @Override
+ public Value getTopValue() {
+ return hasTop() ? keys.get(keyPos).getSecond() : null;
+ }
-
++
+ @Override
+ public void next() throws IOException {
+ // Move on to the next entry since we returned the entry at keyPos before
+ if (keyPos >= 0)
+ keyPos++;
-
++
+ // If we emptied out the transformed key map then transform the next key
+ // set from the source. It’s possible that transformation could produce keys
+ // that are outside of our range or are not visible to the end user, so after the
+ // call below we might not have added any keys to the map. Keep going until
+ // we either get some keys in the map or exhaust the source iterator.
+ while (!hasTop() && super.hasTop())
+ transformKeys();
+ }
-
++
+ @Override
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
+ seekRange = new Range(range);
+ seekColumnFamilies = columnFamilies;
+ seekColumnFamiliesInclusive = inclusive;
-
++
+ // Seek the source iterator, but use a recalculated range that ensures
+ // we see all keys with the same "prefix." We need to do this since
+ // transforming could change the sort order and transformed keys that
+ // are before the range start could be inside the range after transformation.
+ super.seek(computeReseekRange(range), untransformColumnFamilies(columnFamilies), inclusive);
-
++
+ // Range clipping could cause us to trim out all the keys we transformed.
+ // Keep looping until we either have some keys in the output range, or have
+ // exhausted the source iterator.
+ keyPos = -1; // “Clear” list so hasTop returns false to get us into the loop (transformKeys actually clears)
+ while (!hasTop() && super.hasTop()) {
+ // Build up a sorted list of all keys for the same prefix. When
+ // people ask for keys, return from this list first until it is empty
+ // before incrementing the source iterator.
+ transformKeys();
+ }
+ }
-
++
+ private static class RangeIterator implements SortedKeyValueIterator<Key,Value> {
-
++
+ private SortedKeyValueIterator<Key,Value> source;
+ private Key prefixKey;
+ private PartialKey keyPrefix;
+ private boolean hasTop = false;
-
++
+ RangeIterator(SortedKeyValueIterator<Key,Value> source, Key prefixKey, PartialKey keyPrefix) {
+ this.source = source;
+ this.prefixKey = prefixKey;
+ this.keyPrefix = keyPrefix;
+ }
-
++
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ throw new UnsupportedOperationException();
+ }
-
++
+ @Override
+ public boolean hasTop() {
+ // only have a top if the prefix matches
+ return hasTop = source.hasTop() && source.getTopKey().equals(prefixKey, keyPrefix);
+ }
-
++
+ @Override
+ public void next() throws IOException {
+ // do not let user advance too far and try to avoid reexecuting hasTop()
+ if (!hasTop && !hasTop())
+ throw new NoSuchElementException();
+ hasTop = false;
+ source.next();
+ }
-
++
+ @Override
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
+ throw new UnsupportedOperationException();
+ }
-
++
+ @Override
+ public Key getTopKey() {
+ return source.getTopKey();
+ }
-
++
+ @Override
+ public Value getTopValue() {
+ return source.getTopValue();
+ }
-
++
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ throw new UnsupportedOperationException();
+ }
-
++
+ }
-
++
+ /**
+ * Reads all keys matching the first key's prefix from the source iterator, transforms them, and sorts the resulting keys. Transformed keys that fall outside
+ * of our seek range or can't be seen by the user are excluded.
+ */
+ protected void transformKeys() throws IOException {
+ keyPos = -1;
+ keys.clear();
+ final Key prefixKey = super.hasTop() ? new Key(super.getTopKey()) : null;
-
++
+ transformRange(new RangeIterator(getSource(), prefixKey, getKeyPrefix()), new KVBuffer() {
-
++
+ long appened = 0;
-
++
+ @Override
+ public void append(Key key, Value val) {
+ // ensure the key provided by the user has the correct prefix
+ if (!key.equals(prefixKey, getKeyPrefix()))
+ throw new IllegalArgumentException("Key prefixes are not equal " + key + " " + prefixKey);
-
++
+ // Transformation could have produced a key that falls outside
+ // of the seek range, or one that the user cannot see. Check
+ // these before adding it to the output list.
+ if (includeTransformedKey(key)) {
-
++
+ // try to defend against a scan or compaction using all memory in a tablet server
+ if (appened > maxBufferSize)
+ throw new BufferOverflowException("Exceeded buffer size of " + maxBufferSize + ", prefixKey: " + prefixKey);
-
++
+ if (getSource().hasTop() && key == getSource().getTopKey())
+ key = new Key(key);
+ keys.add(new Pair<Key,Value>(key, new Value(val)));
+ appened += (key.getSize() + val.getSize() + 128);
+ }
+ }
+ });
-
++
+ // consume any key in range that user did not consume
+ while (super.hasTop() && super.getTopKey().equals(prefixKey, getKeyPrefix())) {
+ super.next();
+ }
-
++
+ if (!keys.isEmpty()) {
+ Collections.sort(keys, keyComparator);
+ keyPos = 0;
+ }
+ }
-
++
+ /**
+ * Determines whether or not to include {@code transformedKey} in the output. It is possible that transformation could have produced a key that falls outside
+ * of the seek range, a key with a visibility the user can't see, a key with a visibility that doesn't parse, or a key with a column family that wasn't
+ * fetched. We only do some checks (outside the range, user can see) if we're scanning. The range check is not done for major/minor compaction since seek
+ * ranges won't be in our transformed key space and we will never change the row so we can't produce keys that would fall outside the tablet anyway.
- *
++ *
+ * @param transformedKey
+ * the key to check
+ * @return {@code true} if the key should be included and {@code false} if not
+ */
+ protected boolean includeTransformedKey(Key transformedKey) {
+ boolean include = canSee(transformedKey);
+ if (scanning && seekRange != null) {
+ include = include && seekRange.contains(transformedKey);
+ }
+ return include;
+ }
-
++
+ /**
+ * Indicates whether or not the user is able to see {@code key}. If the user has not supplied authorizations, or the iterator is not in the scan scope, then
+ * this method simply returns {@code true}. Otherwise, {@code key}'s column visibility is tested against the user-supplied authorizations, and the test result
+ * is returned. For performance, the test results are cached so that the same visibility is not tested multiple times.
- *
++ *
+ * @param key
+ * the key to test
+ * @return {@code true} if the key is visible or iterator is not scanning, and {@code false} if not
+ */
+ protected boolean canSee(Key key) {
+ // Ensure that the visibility (which could have been transformed) parses. Must always do this check, even if visibility is not evaluated.
+ ByteSequence visibility = key.getColumnVisibilityData();
+ ColumnVisibility colVis = null;
+ Boolean parsed = (Boolean) parsedVisibilitiesCache.get(visibility);
+ if (parsed == null) {
+ try {
+ colVis = new ColumnVisibility(visibility.toArray());
+ parsedVisibilitiesCache.put(visibility, Boolean.TRUE);
+ } catch (BadArgumentException e) {
+ log.error("Parse error after transformation : " + visibility);
+ parsedVisibilitiesCache.put(visibility, Boolean.FALSE);
+ if (scanning) {
+ return false;
+ } else {
+ throw e;
+ }
+ }
+ } else if (!parsed) {
+ if (scanning)
+ return false;
+ else
+ throw new IllegalStateException();
+ }
-
++
+ Boolean visible = canSeeColumnFamily(key);
-
++
+ if (!scanning || !visible || ve == null || visibleCache == null || visibility.length() == 0)
+ return visible;
-
++
+ visible = (Boolean) visibleCache.get(visibility);
+ if (visible == null) {
+ try {
+ if (colVis == null)
+ colVis = new ColumnVisibility(visibility.toArray());
+ visible = ve.evaluate(colVis);
+ visibleCache.put(visibility, visible);
+ } catch (VisibilityParseException e) {
+ log.error("Parse Error", e);
+ visible = Boolean.FALSE;
+ } catch (BadArgumentException e) {
+ log.error("Parse Error", e);
+ visible = Boolean.FALSE;
+ }
+ }
-
++
+ return visible;
+ }
-
++
+ /**
+ * Indicates whether or not {@code key} can be seen, according to the fetched column families for this iterator.
- *
++ *
+ * @param key
+ * the key whose column family is to be tested
+ * @return {@code true} if {@code key}'s column family is one of those fetched in the set passed to our {@link #seek(Range, Collection, boolean)} method
+ */
+ protected boolean canSeeColumnFamily(Key key) {
+ boolean visible = true;
+ if (seekColumnFamilies != null) {
+ ByteSequence columnFamily = key.getColumnFamilyData();
+ if (seekColumnFamiliesInclusive)
+ visible = seekColumnFamilies.contains(columnFamily);
+ else
+ visible = !seekColumnFamilies.contains(columnFamily);
+ }
+ return visible;
+ }
-
++
+ /**
+ * Possibly expand {@code range} to include everything for the key prefix we are working with. That is, if our prefix is ROW_COLFAM, then we need to expand
+ * the range so we're sure to include all entries having the same row and column family as the start/end of the range.
- *
++ *
+ * @param range
+ * the range to expand
+ * @return the modified range
+ */
+ protected Range computeReseekRange(Range range) {
+ Key startKey = range.getStartKey();
+ boolean startKeyInclusive = range.isStartKeyInclusive();
+ // If anything after the prefix is set, then clip the key so we include
+ // everything for the prefix.
+ if (isSetAfterPart(startKey, getKeyPrefix())) {
+ startKey = copyPartialKey(startKey, getKeyPrefix());
+ startKeyInclusive = true;
+ }
+ Key endKey = range.getEndKey();
+ boolean endKeyInclusive = range.isEndKeyInclusive();
+ if (isSetAfterPart(endKey, getKeyPrefix())) {
+ endKey = endKey.followingKey(getKeyPrefix());
+ endKeyInclusive = true;
+ }
+ return new Range(startKey, startKeyInclusive, endKey, endKeyInclusive);
+ }
-
++
+ /**
+ * Indicates whether or not any part of {@code key} excluding {@code part} is set. For example, if part is ROW_COLFAM_COLQUAL, then this method determines
+ * whether or not the column visibility, timestamp, or delete flag is set on {@code key}.
- *
++ *
+ * @param key
+ * the key to check
+ * @param part
+ * the part of the key that doesn't need to be checked (everything after does)
+ * @return {@code true} if anything after {@code part} is set on {@code key}, and {@code false} if not
+ */
+ protected boolean isSetAfterPart(Key key, PartialKey part) {
+ boolean isSet = false;
+ if (key != null) {
+ // Breaks excluded on purpose.
+ switch (part) {
+ case ROW:
+ isSet = isSet || key.getColumnFamilyData().length() > 0;
+ case ROW_COLFAM:
+ isSet = isSet || key.getColumnQualifierData().length() > 0;
+ case ROW_COLFAM_COLQUAL:
+ isSet = isSet || key.getColumnVisibilityData().length() > 0;
+ case ROW_COLFAM_COLQUAL_COLVIS:
+ isSet = isSet || key.getTimestamp() < Long.MAX_VALUE;
+ case ROW_COLFAM_COLQUAL_COLVIS_TIME:
+ isSet = isSet || key.isDeleted();
+ case ROW_COLFAM_COLQUAL_COLVIS_TIME_DEL:
+ break;
+ }
+ }
+ return isSet;
+ }
-
++
+ /**
+ * Creates a copy of {@code key}, copying only the parts of the key specified in {@code part}. For example, if {@code part} is ROW_COLFAM_COLQUAL, then this
+ * method would copy the row, column family, and column qualifier from {@code key} into a new key.
- *
++ *
+ * @param key
+ * the key to copy
+ * @param part
+ * the parts of {@code key} to copy
+ * @return the new key containing {@code part} of {@code key}
+ */
+ protected Key copyPartialKey(Key key, PartialKey part) {
+ Key keyCopy;
+ switch (part) {
+ case ROW:
+ keyCopy = new Key(key.getRow());
+ break;
+ case ROW_COLFAM:
+ keyCopy = new Key(key.getRow(), key.getColumnFamily());
+ break;
+ case ROW_COLFAM_COLQUAL:
+ keyCopy = new Key(key.getRow(), key.getColumnFamily(), key.getColumnQualifier());
+ break;
+ case ROW_COLFAM_COLQUAL_COLVIS:
+ keyCopy = new Key(key.getRow(), key.getColumnFamily(), key.getColumnQualifier(), key.getColumnVisibility());
+ break;
+ case ROW_COLFAM_COLQUAL_COLVIS_TIME:
+ keyCopy = new Key(key.getRow(), key.getColumnFamily(), key.getColumnQualifier(), key.getColumnVisibility(), key.getTimestamp());
+ break;
+ default:
+ throw new IllegalArgumentException("Unsupported key part: " + part);
+ }
+ return keyCopy;
+ }
-
++
+ /**
+ * Make a new key with all parts (including delete flag) coming from {@code originalKey} but use {@code newColFam} as the column family.
+ */
+ protected Key replaceColumnFamily(Key originalKey, Text newColFam) {
+ byte[] row = originalKey.getRowData().toArray();
+ byte[] cf = newColFam.getBytes();
+ byte[] cq = originalKey.getColumnQualifierData().toArray();
+ byte[] cv = originalKey.getColumnVisibilityData().toArray();
+ long timestamp = originalKey.getTimestamp();
+ Key newKey = new Key(row, 0, row.length, cf, 0, newColFam.getLength(), cq, 0, cq.length, cv, 0, cv.length, timestamp);
+ newKey.setDeleted(originalKey.isDeleted());
+ return newKey;
+ }
-
++
+ /**
+ * Make a new key with all parts (including delete flag) coming from {@code originalKey} but use {@code newColQual} as the column qualifier.
+ */
+ protected Key replaceColumnQualifier(Key originalKey, Text newColQual) {
+ byte[] row = originalKey.getRowData().toArray();
+ byte[] cf = originalKey.getColumnFamilyData().toArray();
+ byte[] cq = newColQual.getBytes();
+ byte[] cv = originalKey.getColumnVisibilityData().toArray();
+ long timestamp = originalKey.getTimestamp();
+ Key newKey = new Key(row, 0, row.length, cf, 0, cf.length, cq, 0, newColQual.getLength(), cv, 0, cv.length, timestamp);
+ newKey.setDeleted(originalKey.isDeleted());
+ return newKey;
+ }
-
++
+ /**
+ * Make a new key with all parts (including delete flag) coming from {@code originalKey} but use {@code newColVis} as the column visibility.
+ */
+ protected Key replaceColumnVisibility(Key originalKey, Text newColVis) {
+ byte[] row = originalKey.getRowData().toArray();
+ byte[] cf = originalKey.getColumnFamilyData().toArray();
+ byte[] cq = originalKey.getColumnQualifierData().toArray();
+ byte[] cv = newColVis.getBytes();
+ long timestamp = originalKey.getTimestamp();
+ Key newKey = new Key(row, 0, row.length, cf, 0, cf.length, cq, 0, cq.length, cv, 0, newColVis.getLength(), timestamp);
+ newKey.setDeleted(originalKey.isDeleted());
+ return newKey;
+ }
-
++
+ /**
+ * Make a new key with a column family, column qualifier, and column visibility. Copy the rest of the parts of the key (including delete flag) from
+ * {@code originalKey}.
+ */
+ protected Key replaceKeyParts(Key originalKey, Text newColFam, Text newColQual, Text newColVis) {
+ byte[] row = originalKey.getRowData().toArray();
+ byte[] cf = newColFam.getBytes();
+ byte[] cq = newColQual.getBytes();
+ byte[] cv = newColVis.getBytes();
+ long timestamp = originalKey.getTimestamp();
+ Key newKey = new Key(row, 0, row.length, cf, 0, newColFam.getLength(), cq, 0, newColQual.getLength(), cv, 0, newColVis.getLength(), timestamp);
+ newKey.setDeleted(originalKey.isDeleted());
+ return newKey;
+ }
-
++
+ /**
+ * Make a new key with a column qualifier, and column visibility. Copy the rest of the parts of the key (including delete flag) from {@code originalKey}.
+ */
+ protected Key replaceKeyParts(Key originalKey, Text newColQual, Text newColVis) {
+ byte[] row = originalKey.getRowData().toArray();
+ byte[] cf = originalKey.getColumnFamilyData().toArray();
+ byte[] cq = newColQual.getBytes();
+ byte[] cv = newColVis.getBytes();
+ long timestamp = originalKey.getTimestamp();
+ Key newKey = new Key(row, 0, row.length, cf, 0, cf.length, cq, 0, newColQual.getLength(), cv, 0, newColVis.getLength(), timestamp);
+ newKey.setDeleted(originalKey.isDeleted());
+ return newKey;
+ }
-
++
+ /**
+ * Reverses the transformation applied to column families that are fetched at seek time. If this iterator is transforming column families, then this method
+ * should be overridden to reverse the transformation on the supplied collection of column families. This is necessary since the fetch/seek will be performed
+ * in the transformed space, but when passing the column family set on to the source, the column families need to be in the untransformed space.
- *
++ *
+ * @param columnFamilies
+ * the column families that have been fetched at seek time
+ * @return the untransformed column families that would transform info {@code columnFamilies}
+ */
+ protected Collection<ByteSequence> untransformColumnFamilies(Collection<ByteSequence> columnFamilies) {
+ return columnFamilies;
+ }
-
++
+ /**
+ * Indicates the prefix of keys that will be transformed by this iterator. In other words, this is the part of the key that will <i>not</i> be transformed by
+ * this iterator. For example, if this method returns ROW_COLFAM, then {@link #transformKeys()} may be changing the column qualifier, column visibility, or
+ * timestamp, but it won't be changing the row or column family.
- *
++ *
+ * @return the part of the key this iterator is not transforming
+ */
+ abstract protected PartialKey getKeyPrefix();
-
++
+ public static interface KVBuffer {
+ void append(Key key, Value val);
+ }
-
++
+ /**
+ * Transforms {@code input}. This method must not change the row part of the key, and must only change the parts of the key after the return value of
+ * {@link #getKeyPrefix()}. Implementors must also remember to copy the delete flag from {@code originalKey} onto the new key. Or, implementors should use one
+ * of the helper methods to produce the new key. See any of the replaceKeyParts methods.
- *
++ *
+ * @param input
+ * An iterator over a group of keys with the same prefix. This iterator provides an efficient view, bounded by the prefix, of the underlying iterator
+ * and can not be seeked.
+ * @param output
+ * An output buffer that holds transformed key values. All key values added to the buffer must have the same prefix as the input keys.
- * @throws IOException
+ * @see #replaceColumnFamily(Key, Text)
+ * @see #replaceColumnQualifier(Key, Text)
+ * @see #replaceColumnVisibility(Key, Text)
+ * @see #replaceKeyParts(Key, Text, Text)
+ * @see #replaceKeyParts(Key, Text, Text, Text)
+ */
+ abstract protected void transformRange(SortedKeyValueIterator<Key,Value> input, KVBuffer output) throws IOException;
-
++
+ /**
- * Configure authoriations used for post transformation filtering.
- *
- * @param config
- * @param auths
++ * Configure authorizations used for post transformation filtering.
++ *
+ */
+ public static void setAuthorizations(IteratorSetting config, Authorizations auths) {
+ config.addOption(AUTH_OPT, auths.serialize());
+ }
-
++
+ /**
+ * Configure the maximum amount of memory that can be used for transformation. If this memory is exceeded an exception will be thrown.
- *
- * @param config
++ *
+ * @param maxBufferSize
+ * size in bytes
+ */
+ public static void setMaxBufferSize(IteratorSetting config, long maxBufferSize) {
+ config.addOption(MAX_BUFFER_SIZE_OPT, maxBufferSize + "");
+ }
-
++
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/user/VersioningIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/VersioningIterator.java
index 7804aa4,0000000..2fc3a27
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/VersioningIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/VersioningIterator.java
@@@ -1,172 -1,0 +1,169 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators.user;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.core.iterators.OptionDescriber;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.WrappingIterator;
+
+public class VersioningIterator extends WrappingIterator implements OptionDescriber {
+ private final int maxCount = 10;
+
+ private Key currentKey = new Key();
+ private int numVersions;
+ protected int maxVersions;
+
+ private Range range;
+ private Collection<ByteSequence> columnFamilies;
+ private boolean inclusive;
+
+ @Override
+ public VersioningIterator deepCopy(IteratorEnvironment env) {
+ VersioningIterator copy = new VersioningIterator();
+ copy.setSource(getSource().deepCopy(env));
+ copy.maxVersions = maxVersions;
+ return copy;
+ }
+
+ @Override
+ public void next() throws IOException {
+ if (numVersions >= maxVersions) {
+ skipRowColumn();
+ resetVersionCount();
+ return;
+ }
+
+ super.next();
+ if (getSource().hasTop()) {
+ if (getSource().getTopKey().equals(currentKey, PartialKey.ROW_COLFAM_COLQUAL_COLVIS)) {
+ numVersions++;
+ } else {
+ resetVersionCount();
+ }
+ }
+ }
+
+ @Override
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
+ // do not want to seek to the middle of a row
+ Range seekRange = IteratorUtil.maximizeStartKeyTimeStamp(range);
+ this.range = seekRange;
+ this.columnFamilies = columnFamilies;
+ this.inclusive = inclusive;
+
+ super.seek(seekRange, columnFamilies, inclusive);
+ resetVersionCount();
+
+ if (range.getStartKey() != null)
+ while (hasTop() && range.beforeStartKey(getTopKey()))
+ next();
+ }
+
+ private void resetVersionCount() {
+ if (super.hasTop())
+ currentKey.set(getSource().getTopKey());
+ numVersions = 1;
+ }
+
+ private void skipRowColumn() throws IOException {
+ Key keyToSkip = currentKey;
+ super.next();
+
+ int count = 0;
+ while (getSource().hasTop() && getSource().getTopKey().equals(keyToSkip, PartialKey.ROW_COLFAM_COLQUAL_COLVIS)) {
+ if (count < maxCount) {
+ // it is quicker to call next if we are close, but we never know if we are close
+ // so give next a try a few times
+ getSource().next();
+ count++;
+ } else {
+ reseek(keyToSkip.followingKey(PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
+ count = 0;
+ }
+ }
+ }
+
+ protected void reseek(Key key) throws IOException {
+ if (key == null)
+ return;
+ if (range.afterEndKey(key)) {
+ range = new Range(range.getEndKey(), true, range.getEndKey(), range.isEndKeyInclusive());
+ getSource().seek(range, columnFamilies, inclusive);
+ } else {
+ range = new Range(key, true, range.getEndKey(), range.isEndKeyInclusive());
+ getSource().seek(range, columnFamilies, inclusive);
+ }
+ }
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ super.init(source, options, env);
+ this.numVersions = 0;
+
+ String maxVerString = options.get("maxVersions");
+ if (maxVerString != null)
+ this.maxVersions = Integer.parseInt(maxVerString);
+ else
+ this.maxVersions = 1;
+
+ if (maxVersions < 1)
+ throw new IllegalArgumentException("maxVersions for versioning iterator must be >= 1");
+ }
+
+ @Override
+ public IteratorOptions describeOptions() {
+ return new IteratorOptions("vers", "The VersioningIterator keeps a fixed number of versions for each key", Collections.singletonMap("maxVersions",
+ "number of versions to keep for a particular key (with differing timestamps)"), null);
+ }
+
+ private static final String MAXVERSIONS_OPT = "maxVersions";
+
+ @Override
+ public boolean validateOptions(Map<String,String> options) {
+ int i;
+ try {
+ i = Integer.parseInt(options.get(MAXVERSIONS_OPT));
+ } catch (Exception e) {
+ throw new IllegalArgumentException("bad integer " + MAXVERSIONS_OPT + ":" + options.get(MAXVERSIONS_OPT));
+ }
+ if (i < 1)
+ throw new IllegalArgumentException(MAXVERSIONS_OPT + " for versioning iterator must be >= 1");
+ return true;
+ }
+
+ /**
+ * Encode the maximum number of versions to return onto the ScanIterator
- *
- * @param cfg
- * @param maxVersions
+ */
+ public static void setMaxVersions(IteratorSetting cfg, int maxVersions) {
+ if (maxVersions < 1)
+ throw new IllegalArgumentException(MAXVERSIONS_OPT + " for versioning iterator must be >= 1");
+ cfg.addOption(MAXVERSIONS_OPT, Integer.toString(maxVersions));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/security/SecurityUtil.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/SecurityUtil.java
index 672e784,0000000..65cb7ed
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/security/SecurityUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/SecurityUtil.java
@@@ -1,89 -1,0 +1,88 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.security;
+
+import java.io.IOException;
+import java.net.InetAddress;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.log4j.Logger;
+
+/**
+ *
+ */
+public class SecurityUtil {
+ private static final Logger log = Logger.getLogger(SecurityUtil.class);
+ private static final String ACCUMULO_HOME = "ACCUMULO_HOME", ACCUMULO_CONF_DIR = "ACCUMULO_CONF_DIR";
+ public static boolean usingKerberos = false;
+
+ /**
+ * This method is for logging a server in kerberos. If this is used in client code, it will fail unless run as the accumulo keytab's owner. Instead, use
+ * {@link #login(String, String)}
+ */
+ public static void serverLogin() {
+ @SuppressWarnings("deprecation")
+ AccumuloConfiguration acuConf = AccumuloConfiguration.getSiteConfiguration();
+ String keyTab = acuConf.get(Property.GENERAL_KERBEROS_KEYTAB);
+ if (keyTab == null || keyTab.length() == 0)
+ return;
+
+ usingKerberos = true;
+ if (keyTab.contains("$" + ACCUMULO_HOME) && System.getenv(ACCUMULO_HOME) != null)
+ keyTab = keyTab.replace("$" + ACCUMULO_HOME, System.getenv(ACCUMULO_HOME));
+
+ if (keyTab.contains("$" + ACCUMULO_CONF_DIR) && System.getenv(ACCUMULO_CONF_DIR) != null)
+ keyTab = keyTab.replace("$" + ACCUMULO_CONF_DIR, System.getenv(ACCUMULO_CONF_DIR));
+
+ String principalConfig = acuConf.get(Property.GENERAL_KERBEROS_PRINCIPAL);
+ if (principalConfig == null || principalConfig.length() == 0)
+ return;
+
+ if (login(principalConfig, keyTab)) {
+ try {
+ // This spawns a thread to periodically renew the logged in (accumulo) user
+ UserGroupInformation.getLoginUser();
+ } catch (IOException io) {
+ log.error("Error starting up renewal thread. This shouldn't be happenining.", io);
+ }
+ }
+ }
+
+ /**
+ * This will log in the given user in kerberos.
+ *
+ * @param principalConfig
+ * This is the principals name in the format NAME/HOST@REALM. {@link org.apache.hadoop.security.SecurityUtil#HOSTNAME_PATTERN} will automatically be
+ * replaced by the systems host name.
- * @param keyTabPath
+ * @return true if login succeeded, otherwise false
+ */
+ public static boolean login(String principalConfig, String keyTabPath) {
+ try {
+ String principalName = org.apache.hadoop.security.SecurityUtil.getServerPrincipal(principalConfig, InetAddress.getLocalHost().getCanonicalHostName());
+ if (keyTabPath != null && principalName != null && keyTabPath.length() != 0 && principalName.length() != 0) {
+ UserGroupInformation.loginUserFromKeytab(principalName, keyTabPath);
+ log.info("Succesfully logged in as user " + principalConfig);
+ return true;
+ }
+ } catch (IOException io) {
+ log.error("Error logging in user " + principalConfig + " using keytab at " + keyTabPath, io);
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModule.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModule.java
index fca7d22,0000000..40d3ab7
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModule.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModule.java
@@@ -1,110 -1,0 +1,108 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.security.crypto;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Map;
+
+/**
+ * Classes that obey this interface may be used to provide encrypting and decrypting streams to the rest of Accumulo. Classes that obey this interface may be
+ * configured as the crypto module by setting the property crypto.module.class in the accumulo-site.xml file.
+ *
+ * Note that this first iteration of this API is considered deprecated because we anticipate it changing in non-backwards compatible ways as we explore the
+ * requirements for encryption in Accumulo. So, your mileage is gonna vary a lot as we go forward.
+ *
+ */
+@Deprecated
+public interface CryptoModule {
+
+ public enum CryptoInitProperty {
+ ALGORITHM_NAME("algorithm.name"), CIPHER_SUITE("cipher.suite"), INITIALIZATION_VECTOR("initialization.vector"), PLAINTEXT_SESSION_KEY(
+ "plaintext.session.key");
+
+ private CryptoInitProperty(String name) {
+ key = name;
+ }
+
+ private String key;
+
+ public String getKey() {
+ return key;
+ }
+ }
+
+ /**
+ * Wraps an OutputStream in an encrypting OutputStream. The given map contains the settings for the cryptographic algorithm to use. <b>Callers of this method
+ * should expect that the given OutputStream will be written to before cryptographic writes occur.</b> These writes contain the cryptographic information used
+ * to encrypt the following bytes (these data include the initialization vector, encrypted session key, and so on). If writing arbitrarily to the underlying
+ * stream is not desirable, users should call the other flavor of getEncryptingOutputStream which accepts these data as parameters.
+ *
+ * @param out
+ * the OutputStream to wrap
+ * @param cryptoOpts
+ * the cryptographic parameters to use; specific string names to look for will depend on the various implementations
+ * @return an OutputStream that wraps the given parameter
- * @throws IOException
+ */
+ public OutputStream getEncryptingOutputStream(OutputStream out, Map<String,String> cryptoOpts) throws IOException;
+
+ /**
+ * Wraps an InputStream and returns a decrypting input stream. The given map contains the settings for the intended cryptographic operations, but implementors
+ * should take care to ensure that the crypto from the given input stream matches their expectations about what they will use to decrypt it, as the parameters
+ * may have changed. Also, care should be taken around transitioning between non-encrypting and encrypting streams; implementors should handle the case where
+ * the given input stream is <b>not</b> encrypted at all.
+ *
+ * It is expected that this version of getDecryptingInputStream is called in conjunction with the getEncryptingOutputStream from above. It should expect its
+ * input streams to contain the data written by getEncryptingOutputStream.
+ *
+ * @param in
+ * the InputStream to wrap
+ * @param cryptoOpts
+ * the cryptographic parameters to use; specific string names to look for will depend on the various implementations
+ * @return an InputStream that wraps the given parameter
- * @throws IOException
+ */
+ public InputStream getDecryptingInputStream(InputStream in, Map<String,String> cryptoOpts) throws IOException;
+
+ /**
+ * Wraps an OutputStream in an encrypting OutputStream. The given map contains the settings for the cryptographic algorithm to use. The cryptoInitParams map
+ * contains all the cryptographic details to construct a key (or keys), initialization vectors, etc. and use them to properly initialize the stream for
+ * writing. These initialization parameters must be persisted elsewhere, along with the cryptographic configuration (algorithm, mode, etc.), so that they may
+ * be read in at the time of reading the encrypted content.
+ *
+ * @param out
+ * the OutputStream to wrap
+ * @param conf
+ * the cryptographic algorithm configuration
+ * @param cryptoInitParams
+ * the initialization parameters for the algorithm, usually including initialization vector and session key
+ * @return a wrapped output stream
+ */
+ public OutputStream getEncryptingOutputStream(OutputStream out, Map<String,String> conf, Map<CryptoModule.CryptoInitProperty,Object> cryptoInitParams);
+
+ /**
+ * Wraps an InputStream and returns a decrypting input stream. The given map contains the settings for the intended cryptographic operations, but implementors
+ * should take care to ensure that the crypto from the given input stream matches their expectations about what they will use to decrypt it, as the parameters
+ * may have changed. Also, care should be taken around transitioning between non-encrypting and encrypting streams; implementors should handle the case where
+ * the given input stream is <b>not</b> encrypted at all.
+ *
+ * The cryptoInitParams contains all necessary information to properly initialize the given cipher, usually including things like initialization vector and
+ * secret key.
+ */
+ public InputStream getDecryptingInputStream(InputStream in, Map<String,String> cryptoOpts, Map<CryptoModule.CryptoInitProperty,Object> cryptoInitParams)
+ throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleFactory.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleFactory.java
index 2f03e02,0000000..956c961
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleFactory.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleFactory.java
@@@ -1,254 -1,0 +1,253 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.security.crypto;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Map;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.log4j.Logger;
+
+/**
+ * This factory module exists to assist other classes in loading crypto modules.
+ *
+ * @deprecated This feature is experimental and may go away in future versions.
+ */
+@Deprecated
+public class CryptoModuleFactory {
+
+ private static Logger log = Logger.getLogger(CryptoModuleFactory.class);
+
+ /**
+ * This method returns a crypto module based on settings in the given configuration parameter.
+ *
- * @param conf
+ * @return a class implementing the CryptoModule interface. It will *never* return null; rather, it will return a class which obeys the interface but makes no
+ * changes to the underlying data.
+ */
+ public static CryptoModule getCryptoModule(AccumuloConfiguration conf) {
+ String cryptoModuleClassname = conf.get(Property.CRYPTO_MODULE_CLASS);
+ return getCryptoModule(cryptoModuleClassname);
+ }
+
+ @SuppressWarnings({"rawtypes"})
+ public static CryptoModule getCryptoModule(String cryptoModuleClassname) {
+ log.debug(String.format("About to instantiate crypto module %s", cryptoModuleClassname));
+
+ if (cryptoModuleClassname.equals("NullCryptoModule")) {
+ return new NullCryptoModule();
+ }
+
+ CryptoModule cryptoModule = null;
+ Class cryptoModuleClazz = null;
+ try {
+ cryptoModuleClazz = AccumuloVFSClassLoader.loadClass(cryptoModuleClassname);
+ } catch (ClassNotFoundException e1) {
+ log.warn(String.format("Could not find configured crypto module \"%s\". NO ENCRYPTION WILL BE USED.", cryptoModuleClassname));
+ return new NullCryptoModule();
+ }
+
+ // Check if the given class implements the CryptoModule interface
+ Class[] interfaces = cryptoModuleClazz.getInterfaces();
+ boolean implementsCryptoModule = false;
+
+ for (Class clazz : interfaces) {
+ if (clazz.equals(CryptoModule.class)) {
+ implementsCryptoModule = true;
+ break;
+ }
+ }
+
+ if (!implementsCryptoModule) {
+ log.warn("Configured Accumulo crypto module \"%s\" does not implement the CryptoModule interface. NO ENCRYPTION WILL BE USED.");
+ return new NullCryptoModule();
+ } else {
+ try {
+ cryptoModule = (CryptoModule) cryptoModuleClazz.newInstance();
+
+ log.debug("Successfully instantiated crypto module");
+
+ } catch (InstantiationException e) {
+ log.warn(String.format("Got instantiation exception %s when instantiating crypto module \"%s\". NO ENCRYPTION WILL BE USED.", e.getCause().getClass()
+ .getCanonicalName(), cryptoModuleClassname));
+ log.warn(e.getCause());
+ return new NullCryptoModule();
+ } catch (IllegalAccessException e) {
+ log.warn(String.format("Got illegal access exception when trying to instantiate crypto module \"%s\". NO ENCRYPTION WILL BE USED.",
+ cryptoModuleClassname));
+ log.warn(e);
+ return new NullCryptoModule();
+ }
+ }
+ return cryptoModule;
+ }
+
+ public static SecretKeyEncryptionStrategy getSecretKeyEncryptionStrategy(AccumuloConfiguration conf) {
+ String className = conf.get(Property.CRYPTO_SECRET_KEY_ENCRYPTION_STRATEGY_CLASS);
+ return getSecretKeyEncryptionStrategy(className);
+ }
+
+ @SuppressWarnings("rawtypes")
+ public static SecretKeyEncryptionStrategy getSecretKeyEncryptionStrategy(String className) {
+ if (className == null || className.equals("NullSecretKeyEncryptionStrategy")) {
+ return new NullSecretKeyEncryptionStrategy();
+ }
+
+ SecretKeyEncryptionStrategy strategy = null;
+ Class keyEncryptionStrategyClazz = null;
+ try {
+ keyEncryptionStrategyClazz = AccumuloVFSClassLoader.loadClass(className);
+ } catch (ClassNotFoundException e1) {
+ log.warn(String.format("Could not find configured secret key encryption strategy \"%s\". NO ENCRYPTION WILL BE USED.", className));
+ return new NullSecretKeyEncryptionStrategy();
+ }
+
+ // Check if the given class implements the CryptoModule interface
+ Class[] interfaces = keyEncryptionStrategyClazz.getInterfaces();
+ boolean implementsSecretKeyStrategy = false;
+
+ for (Class clazz : interfaces) {
+ if (clazz.equals(SecretKeyEncryptionStrategy.class)) {
+ implementsSecretKeyStrategy = true;
+ break;
+ }
+ }
+
+ if (!implementsSecretKeyStrategy) {
+ log.warn("Configured Accumulo secret key encryption strategy \"%s\" does not implement the SecretKeyEncryptionStrategy interface. NO ENCRYPTION WILL BE USED.");
+ return new NullSecretKeyEncryptionStrategy();
+ } else {
+ try {
+ strategy = (SecretKeyEncryptionStrategy) keyEncryptionStrategyClazz.newInstance();
+
+ log.debug("Successfully instantiated secret key encryption strategy");
+
+ } catch (InstantiationException e) {
+ log.warn(String.format("Got instantiation exception %s when instantiating secret key encryption strategy \"%s\". NO ENCRYPTION WILL BE USED.", e
+ .getCause().getClass().getCanonicalName(), className));
+ log.warn(e.getCause());
+ return new NullSecretKeyEncryptionStrategy();
+ } catch (IllegalAccessException e) {
+ log.warn(String.format("Got illegal access exception when trying to instantiate secret key encryption strategy \"%s\". NO ENCRYPTION WILL BE USED.",
+ className));
+ log.warn(e);
+ return new NullSecretKeyEncryptionStrategy();
+ }
+ }
+
+ return strategy;
+ }
+
+ private static class NullSecretKeyEncryptionStrategy implements SecretKeyEncryptionStrategy {
+
+ @Override
+ public SecretKeyEncryptionStrategyContext encryptSecretKey(SecretKeyEncryptionStrategyContext context) {
+ context.setEncryptedSecretKey(context.getPlaintextSecretKey());
+ context.setOpaqueKeyEncryptionKeyID("");
+
+ return context;
+ }
+
+ @Override
+ public SecretKeyEncryptionStrategyContext decryptSecretKey(SecretKeyEncryptionStrategyContext context) {
+ context.setPlaintextSecretKey(context.getEncryptedSecretKey());
+
+ return context;
+ }
+
+ @Override
+ public SecretKeyEncryptionStrategyContext getNewContext() {
+ return new SecretKeyEncryptionStrategyContext() {
+
+ @Override
+ public byte[] getPlaintextSecretKey() {
+ return plaintextSecretKey;
+ }
+
+ @Override
+ public void setPlaintextSecretKey(byte[] plaintextSecretKey) {
+ this.plaintextSecretKey = plaintextSecretKey;
+ }
+
+ @Override
+ public byte[] getEncryptedSecretKey() {
+ return encryptedSecretKey;
+ }
+
+ @Override
+ public void setEncryptedSecretKey(byte[] encryptedSecretKey) {
+ this.encryptedSecretKey = encryptedSecretKey;
+ }
+
+ @Override
+ public String getOpaqueKeyEncryptionKeyID() {
+ return opaqueKeyEncryptionKeyID;
+ }
+
+ @Override
+ public void setOpaqueKeyEncryptionKeyID(String opaqueKeyEncryptionKeyID) {
+ this.opaqueKeyEncryptionKeyID = opaqueKeyEncryptionKeyID;
+ }
+
+ @Override
+ public Map<String,String> getContext() {
+ return context;
+ }
+
+ @Override
+ public void setContext(Map<String,String> context) {
+ this.context = context;
+ }
+
+ private byte[] plaintextSecretKey;
+ private byte[] encryptedSecretKey;
+ private String opaqueKeyEncryptionKeyID;
+ private Map<String,String> context;
+ };
+ }
+
+ }
+
+ private static class NullCryptoModule implements CryptoModule {
+
+ @Override
+ public OutputStream getEncryptingOutputStream(OutputStream out, Map<String,String> cryptoOpts) throws IOException {
+ return out;
+ }
+
+ @Override
+ public InputStream getDecryptingInputStream(InputStream in, Map<String,String> cryptoOpts) throws IOException {
+ return in;
+ }
+
+ @Override
+ public OutputStream getEncryptingOutputStream(OutputStream out, Map<String,String> conf, Map<CryptoInitProperty,Object> cryptoInitParams) {
+ return out;
+ }
+
+ @Override
+ public InputStream getDecryptingInputStream(InputStream in, Map<String,String> cryptoOpts, Map<CryptoInitProperty,Object> cryptoInitParams)
+ throws IOException {
+ return in;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/test/java/org/apache/accumulo/core/client/impl/ScannerOptionsTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/impl/ScannerOptionsTest.java
index 07bd518,0000000..463822a
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/impl/ScannerOptionsTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/impl/ScannerOptionsTest.java
@@@ -1,59 -1,0 +1,57 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.impl;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.iterators.DebugIterator;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.junit.Test;
+
+/**
+ * Test that scanner options are set/unset correctly
+ */
+public class ScannerOptionsTest {
+
+ /**
+ * Test that you properly add and remove iterators from a scanner
- *
- * @throws Throwable
+ */
+ @Test
+ public void testAddRemoveIterator() throws Throwable {
+ ScannerOptions options = new ScannerOptions();
+ options.addScanIterator(new IteratorSetting(1, "NAME", WholeRowIterator.class));
+ assertEquals(1, options.serverSideIteratorList.size());
+ options.removeScanIterator("NAME");
+ assertEquals(0, options.serverSideIteratorList.size());
+ }
+
+ @Test
+ public void testIteratorConflict() {
+ ScannerOptions options = new ScannerOptions();
+ options.addScanIterator(new IteratorSetting(1, "NAME", DebugIterator.class));
+ try {
+ options.addScanIterator(new IteratorSetting(2, "NAME", DebugIterator.class));
+ fail();
+ } catch (IllegalArgumentException e) {}
+ try {
+ options.addScanIterator(new IteratorSetting(1, "NAME2", DebugIterator.class));
+ fail();
+ } catch (IllegalArgumentException e) {}
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
index 4f527e1,0000000..3ec9bb1
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
@@@ -1,284 -1,0 +1,280 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapred;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.lib.NullOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.Test;
+
+public class AccumuloInputFormatTest {
+
+ private static final String PREFIX = AccumuloInputFormatTest.class.getSimpleName();
+ private static final String INSTANCE_NAME = PREFIX + "_mapred_instance";
+ private static final String TEST_TABLE_1 = PREFIX + "_mapred_table_1";
+
+ /**
+ * Check that the iterator configuration is getting stored in the Job conf correctly.
- *
- * @throws IOException
+ */
+ @Test
+ public void testSetIterator() throws IOException {
+ JobConf job = new JobConf();
+
+ IteratorSetting is = new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator");
+ AccumuloInputFormat.addIterator(job, is);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ is.write(new DataOutputStream(baos));
+ String iterators = job.get("AccumuloInputFormat.ScanOpts.Iterators");
+ assertEquals(new String(Base64.encodeBase64(baos.toByteArray())), iterators);
+ }
+
+ @Test
+ public void testAddIterator() throws IOException {
+ JobConf job = new JobConf();
+
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", WholeRowIterator.class));
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
+ IteratorSetting iter = new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator");
+ iter.addOption("v1", "1");
+ iter.addOption("junk", "\0omg:!\\xyzzy");
+ AccumuloInputFormat.addIterator(job, iter);
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+
+ // Check the list size
+ assertTrue(list.size() == 3);
+
+ // Walk the list and make sure our settings are correct
+ IteratorSetting setting = list.get(0);
+ assertEquals(1, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.user.WholeRowIterator", setting.getIteratorClass());
+ assertEquals("WholeRow", setting.getName());
+ assertEquals(0, setting.getOptions().size());
+
+ setting = list.get(1);
+ assertEquals(2, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
+ assertEquals("Versions", setting.getName());
+ assertEquals(0, setting.getOptions().size());
+
+ setting = list.get(2);
+ assertEquals(3, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
+ assertEquals("Count", setting.getName());
+ assertEquals(2, setting.getOptions().size());
+ assertEquals("1", setting.getOptions().get("v1"));
+ assertEquals("\0omg:!\\xyzzy", setting.getOptions().get("junk"));
+ }
+
+ /**
+ * Test adding iterator options where the keys and values contain both the FIELD_SEPARATOR character (':') and ITERATOR_SEPARATOR (',') characters. There
+ * should be no exceptions thrown when trying to parse these types of option entries.
+ *
+ * This test makes sure that the expected raw values, as appears in the Job, are equal to what's expected.
+ */
+ @Test
+ public void testIteratorOptionEncoding() throws Throwable {
+ String key = "colon:delimited:key";
+ String value = "comma,delimited,value";
+ IteratorSetting someSetting = new IteratorSetting(1, "iterator", "Iterator.class");
+ someSetting.addOption(key, value);
+ JobConf job = new JobConf();
+ AccumuloInputFormat.addIterator(job, someSetting);
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+ assertEquals(1, list.size());
+ assertEquals(1, list.get(0).getOptions().size());
+ assertEquals(list.get(0).getOptions().get(key), value);
+
+ someSetting.addOption(key + "2", value);
+ someSetting.setPriority(2);
+ someSetting.setName("it2");
+ AccumuloInputFormat.addIterator(job, someSetting);
+ list = AccumuloInputFormat.getIterators(job);
+ assertEquals(2, list.size());
+ assertEquals(1, list.get(0).getOptions().size());
+ assertEquals(list.get(0).getOptions().get(key), value);
+ assertEquals(2, list.get(1).getOptions().size());
+ assertEquals(list.get(1).getOptions().get(key), value);
+ assertEquals(list.get(1).getOptions().get(key + "2"), value);
+ }
+
+ /**
+ * Test getting iterator settings for multiple iterators set
- *
- * @throws IOException
+ */
+ @Test
+ public void testGetIteratorSettings() throws IOException {
+ JobConf job = new JobConf();
+
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator"));
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator"));
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+
+ // Check the list size
+ assertTrue(list.size() == 3);
+
+ // Walk the list and make sure our settings are correct
+ IteratorSetting setting = list.get(0);
+ assertEquals(1, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.WholeRowIterator", setting.getIteratorClass());
+ assertEquals("WholeRow", setting.getName());
+
+ setting = list.get(1);
+ assertEquals(2, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
+ assertEquals("Versions", setting.getName());
+
+ setting = list.get(2);
+ assertEquals(3, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
+ assertEquals("Count", setting.getName());
+
+ }
+
+ @Test
+ public void testSetRegex() throws IOException {
+ JobConf job = new JobConf();
+
+ String regex = ">\"*%<>\'\\";
+
+ IteratorSetting is = new IteratorSetting(50, regex, RegExFilter.class);
+ RegExFilter.setRegexs(is, regex, null, null, null, false);
+ AccumuloInputFormat.addIterator(job, is);
+
+ assertTrue(regex.equals(AccumuloInputFormat.getIterators(job).get(0).getName()));
+ }
+
+ private static AssertionError e1 = null;
+ private static AssertionError e2 = null;
+
+ private static class MRTester extends Configured implements Tool {
+ private static class TestMapper implements Mapper<Key,Value,Key,Value> {
+ Key key = null;
+ int count = 0;
+
+ @Override
+ public void map(Key k, Value v, OutputCollector<Key,Value> output, Reporter reporter) throws IOException {
+ try {
+ if (key != null)
+ assertEquals(key.getRow().toString(), new String(v.get()));
+ assertEquals(k.getRow(), new Text(String.format("%09x", count + 1)));
+ assertEquals(new String(v.get()), String.format("%09x", count));
+ } catch (AssertionError e) {
+ e1 = e;
+ }
+ key = new Key(k);
+ count++;
+ }
+
+ @Override
+ public void configure(JobConf job) {}
+
+ @Override
+ public void close() throws IOException {
+ try {
+ assertEquals(100, count);
+ } catch (AssertionError e) {
+ e2 = e;
+ }
+ }
+
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+
+ if (args.length != 3) {
+ throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <user> <pass> <table>");
+ }
+
+ String user = args[0];
+ String pass = args[1];
+ String table = args[2];
+
+ JobConf job = new JobConf(getConf());
+ job.setJarByClass(this.getClass());
+
+ job.setInputFormat(AccumuloInputFormat.class);
+
+ AccumuloInputFormat.setConnectorInfo(job, user, new PasswordToken(pass));
+ AccumuloInputFormat.setInputTableName(job, table);
+ AccumuloInputFormat.setMockInstance(job, INSTANCE_NAME);
+
+ job.setMapperClass(TestMapper.class);
+ job.setMapOutputKeyClass(Key.class);
+ job.setMapOutputValueClass(Value.class);
+ job.setOutputFormat(NullOutputFormat.class);
+
+ job.setNumReduceTasks(0);
+
+ return JobClient.runJob(job).isSuccessful() ? 0 : 1;
+ }
+
+ public static void main(String[] args) throws Exception {
+ assertEquals(0, ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args));
+ }
+ }
+
+ @Test
+ public void testMap() throws Exception {
+ MockInstance mockInstance = new MockInstance(INSTANCE_NAME);
+ Connector c = mockInstance.getConnector("root", new PasswordToken(""));
+ c.tableOperations().create(TEST_TABLE_1);
+ BatchWriter bw = c.createBatchWriter(TEST_TABLE_1, new BatchWriterConfig());
+ for (int i = 0; i < 100; i++) {
+ Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
+ m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
+ bw.addMutation(m);
+ }
+ bw.close();
+
+ MRTester.main(new String[] {"root", "", TEST_TABLE_1});
+ assertNull(e1);
+ assertNull(e2);
+ }
+}
[10/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFile.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFile.java
index f2cb326,0000000..b7a927b
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFile.java
@@@ -1,2030 -1,0 +1,1986 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.accumulo.core.file.rfile.bcfile;
+
+import java.io.ByteArrayInputStream;
+import java.io.Closeable;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Comparator;
+
+import org.apache.accumulo.core.file.rfile.bcfile.BCFile.Reader.BlockReader;
+import org.apache.accumulo.core.file.rfile.bcfile.BCFile.Writer.BlockAppender;
+import org.apache.accumulo.core.file.rfile.bcfile.Chunk.ChunkDecoder;
+import org.apache.accumulo.core.file.rfile.bcfile.Chunk.ChunkEncoder;
+import org.apache.accumulo.core.file.rfile.bcfile.CompareUtils.BytesComparator;
+import org.apache.accumulo.core.file.rfile.bcfile.CompareUtils.MemcmpRawComparator;
+import org.apache.accumulo.core.file.rfile.bcfile.Utils.Version;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.RawComparator;
+import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.io.serializer.JavaSerializationComparator;
+
+/**
+ * A TFile is a container of key-value pairs. Both keys and values are type-less bytes. Keys are restricted to 64KB, value length is not restricted (practically
+ * limited to the available disk storage). TFile further provides the following features:
+ * <ul>
+ * <li>Block Compression.
+ * <li>Named meta data blocks.
+ * <li>Sorted or unsorted keys.
+ * <li>Seek by key or by file offset.
+ * </ul>
+ * The memory footprint of a TFile includes the following:
+ * <ul>
+ * <li>Some constant overhead of reading or writing a compressed block.
+ * <ul>
+ * <li>Each compressed block requires one compression/decompression codec for I/O.
+ * <li>Temporary space to buffer the key.
+ * <li>Temporary space to buffer the value (for TFile.Writer only). Values are chunk encoded, so that we buffer at most one chunk of user data. By default, the
+ * chunk buffer is 1MB. Reading chunked value does not require additional memory.
+ * </ul>
+ * <li>TFile index, which is proportional to the total number of Data Blocks. The total amount of memory needed to hold the index can be estimated as
+ * (56+AvgKeySize)*NumBlocks.
+ * <li>MetaBlock index, which is proportional to the total number of Meta Blocks.The total amount of memory needed to hold the index for Meta Blocks can be
+ * estimated as (40+AvgMetaBlockName)*NumMetaBlock.
+ * </ul>
+ * <p>
+ * The behavior of TFile can be customized by the following variables through Configuration:
+ * <ul>
+ * <li><b>tfile.io.chunk.size</b>: Value chunk size. Integer (in bytes). Default to 1MB. Values of the length less than the chunk size is guaranteed to have
+ * known value length in read time (See {@link TFile.Reader.Scanner.Entry#isValueLengthKnown()}).
+ * <li><b>tfile.fs.output.buffer.size</b>: Buffer size used for FSDataOutputStream. Integer (in bytes). Default to 256KB.
+ * <li><b>tfile.fs.input.buffer.size</b>: Buffer size used for FSDataInputStream. Integer (in bytes). Default to 256KB.
+ * </ul>
+ * <p>
+ * Suggestions on performance optimization.
+ * <ul>
+ * <li>Minimum block size. We recommend a setting of minimum block size between 256KB to 1MB for general usage. Larger block size is preferred if files are
+ * primarily for sequential access. However, it would lead to inefficient random access (because there are more data to decompress). Smaller blocks are good for
+ * random access, but require more memory to hold the block index, and may be slower to create (because we must flush the compressor stream at the conclusion of
+ * each data block, which leads to an FS I/O flush). Further, due to the internal caching in Compression codec, the smallest possible block size would be around
+ * 20KB-30KB.
+ * <li>The current implementation does not offer true multi-threading for reading. The implementation uses FSDataInputStream seek()+read(), which is shown to be
+ * much faster than positioned-read call in single thread mode. However, it also means that if multiple threads attempt to access the same TFile (using multiple
+ * scanners) simultaneously, the actual I/O is carried out sequentially even if they access different DFS blocks.
+ * <li>Compression codec. Use "none" if the data is not very compressable (by compressable, I mean a compression ratio at least 2:1). Generally, use "lzo" as
+ * the starting point for experimenting. "gz" overs slightly better compression ratio over "lzo" but requires 4x CPU to compress and 2x CPU to decompress,
+ * comparing to "lzo".
+ * <li>File system buffering, if the underlying FSDataInputStream and FSDataOutputStream is already adequately buffered; or if applications reads/writes keys
+ * and values in large buffers, we can reduce the sizes of input/output buffering in TFile layer by setting the configuration parameters
+ * "tfile.fs.input.buffer.size" and "tfile.fs.output.buffer.size".
+ * </ul>
+ *
+ * Some design rationale behind TFile can be found at <a href=https://issues.apache.org/jira/browse/HADOOP-3315>Hadoop-3315</a>.
+ */
+public class TFile {
+ static final Log LOG = LogFactory.getLog(TFile.class);
+
+ private static final String CHUNK_BUF_SIZE_ATTR = "tfile.io.chunk.size";
+ private static final String FS_INPUT_BUF_SIZE_ATTR = "tfile.fs.input.buffer.size";
+ private static final String FS_OUTPUT_BUF_SIZE_ATTR = "tfile.fs.output.buffer.size";
+
+ static int getChunkBufferSize(Configuration conf) {
+ int ret = conf.getInt(CHUNK_BUF_SIZE_ATTR, 1024 * 1024);
+ return (ret > 0) ? ret : 1024 * 1024;
+ }
+
+ static int getFSInputBufferSize(Configuration conf) {
+ return conf.getInt(FS_INPUT_BUF_SIZE_ATTR, 256 * 1024);
+ }
+
+ static int getFSOutputBufferSize(Configuration conf) {
+ return conf.getInt(FS_OUTPUT_BUF_SIZE_ATTR, 256 * 1024);
+ }
+
+ private static final int MAX_KEY_SIZE = 64 * 1024; // 64KB
+ static final Version API_VERSION = new Version((short) 1, (short) 0);
+
+ /** snappy codec **/
+ public static final String COMPRESSION_SNAPPY = "snappy";
+
+ /** compression: gzip */
+ public static final String COMPRESSION_GZ = "gz";
+ /** compression: lzo */
+ public static final String COMPRESSION_LZO = "lzo";
+ /** compression: none */
+ public static final String COMPRESSION_NONE = "none";
+ /** comparator: memcmp */
+ public static final String COMPARATOR_MEMCMP = "memcmp";
+ /** comparator prefix: java class */
+ public static final String COMPARATOR_JCLASS = "jclass:";
+
+ /**
+ * Make a raw comparator from a string name.
+ *
+ * @param name
+ * Comparator name
+ * @return A RawComparable comparator.
+ */
+ static public Comparator<RawComparable> makeComparator(String name) {
+ return TFileMeta.makeComparator(name);
+ }
+
+ // Prevent the instantiation of TFiles
+ private TFile() {
+ // nothing
+ }
+
+ /**
+ * Get names of supported compression algorithms. The names are acceptable by TFile.Writer.
+ *
+ * @return Array of strings, each represents a supported compression algorithm. Currently, the following compression algorithms are supported.
+ * <ul>
+ * <li>"none" - No compression.
+ * <li>"lzo" - LZO compression.
+ * <li>"gz" - GZIP compression.
+ * <li>"snappy" - Snappy compression
+ * </ul>
+ */
+ public static String[] getSupportedCompressionAlgorithms() {
+ return Compression.getSupportedAlgorithms();
+ }
+
+ /**
+ * TFile Writer.
+ */
+ public static class Writer implements Closeable {
+ // minimum compressed size for a block.
+ private final int sizeMinBlock;
+
+ // Meta blocks.
+ final TFileIndex tfileIndex;
+ final TFileMeta tfileMeta;
+
+ // reference to the underlying BCFile.
+ private BCFile.Writer writerBCF;
+
+ // current data block appender.
+ BlockAppender blkAppender;
+ long blkRecordCount;
+
+ // buffers for caching the key.
+ BoundedByteArrayOutputStream currentKeyBufferOS;
+ BoundedByteArrayOutputStream lastKeyBufferOS;
+
+ // buffer used by chunk codec
+ private byte[] valueBuffer;
+
+ /**
+ * Writer states. The state always transits in circles: READY -> IN_KEY -> END_KEY -> IN_VALUE -> READY.
+ */
+ private enum State {
+ READY, // Ready to start a new key-value pair insertion.
+ IN_KEY, // In the middle of key insertion.
+ END_KEY, // Key insertion complete, ready to insert value.
+ IN_VALUE, // In value insertion.
+ // ERROR, // Error encountered, cannot continue.
+ CLOSED, // TFile already closed.
+ }
+
+ // current state of Writer.
+ State state = State.READY;
+ Configuration conf;
+ long errorCount = 0;
+
+ /**
+ * Constructor
+ *
+ * @param fsdos
+ * output stream for writing. Must be at position 0.
+ * @param minBlockSize
+ * Minimum compressed block size in bytes. A compression block will not be closed until it reaches this size except for the last block.
+ * @param compressName
+ * Name of the compression algorithm. Must be one of the strings returned by {@link TFile#getSupportedCompressionAlgorithms()}.
+ * @param comparator
+ * Leave comparator as null or empty string if TFile is not sorted. Otherwise, provide the string name for the comparison algorithm for keys. Two
+ * kinds of comparators are supported.
+ * <ul>
+ * <li>Algorithmic comparator: binary comparators that is language independent. Currently, only "memcmp" is supported.
+ * <li>Language-specific comparator: binary comparators that can only be constructed in specific language. For Java, the syntax is "jclass:",
+ * followed by the class name of the RawComparator. Currently, we only support RawComparators that can be constructed through the default
+ * constructor (with no parameters). Parameterized RawComparators such as {@link WritableComparator} or {@link JavaSerializationComparator} may not
+ * be directly used. One should write a wrapper class that inherits from such classes and use its default constructor to perform proper
+ * initialization.
+ * </ul>
+ * @param conf
+ * The configuration object.
- * @throws IOException
+ */
+ public Writer(FSDataOutputStream fsdos, int minBlockSize, String compressName, String comparator, Configuration conf) throws IOException {
+ sizeMinBlock = minBlockSize;
+ tfileMeta = new TFileMeta(comparator);
+ tfileIndex = new TFileIndex(tfileMeta.getComparator());
+
+ writerBCF = new BCFile.Writer(fsdos, compressName, conf, true);
+ currentKeyBufferOS = new BoundedByteArrayOutputStream(MAX_KEY_SIZE);
+ lastKeyBufferOS = new BoundedByteArrayOutputStream(MAX_KEY_SIZE);
+ this.conf = conf;
+ }
+
+ /**
+ * Close the Writer. Resources will be released regardless of the exceptions being thrown. Future close calls will have no effect.
+ *
+ * The underlying FSDataOutputStream is not closed.
+ */
+ public void close() throws IOException {
+ if ((state == State.CLOSED)) {
+ return;
+ }
+ try {
+ // First try the normal finish.
+ // Terminate upon the first Exception.
+ if (errorCount == 0) {
+ if (state != State.READY) {
+ throw new IllegalStateException("Cannot close TFile in the middle of key-value insertion.");
+ }
+
+ finishDataBlock(true);
+
+ // first, write out data:TFile.meta
+ BlockAppender outMeta = writerBCF.prepareMetaBlock(TFileMeta.BLOCK_NAME, COMPRESSION_NONE);
+ try {
+ tfileMeta.write(outMeta);
+ } finally {
+ outMeta.close();
+ }
+
+ // second, write out data:TFile.index
+ BlockAppender outIndex = writerBCF.prepareMetaBlock(TFileIndex.BLOCK_NAME);
+ try {
+ tfileIndex.write(outIndex);
+ } finally {
+ outIndex.close();
+ }
+
+ writerBCF.close();
+ }
+ } finally {
+ IOUtils.cleanup(LOG, blkAppender, writerBCF);
+ blkAppender = null;
+ writerBCF = null;
+ state = State.CLOSED;
+ }
+ }
+
+ /**
+ * Adding a new key-value pair to the TFile. This is synonymous to append(key, 0, key.length, value, 0, value.length)
+ *
+ * @param key
+ * Buffer for key.
+ * @param value
+ * Buffer for value.
- * @throws IOException
+ */
+ public void append(byte[] key, byte[] value) throws IOException {
+ append(key, 0, key.length, value, 0, value.length);
+ }
+
+ /**
+ * Adding a new key-value pair to TFile.
+ *
+ * @param key
+ * buffer for key.
+ * @param koff
+ * offset in key buffer.
+ * @param klen
+ * length of key.
+ * @param value
+ * buffer for value.
+ * @param voff
+ * offset in value buffer.
+ * @param vlen
+ * length of value.
+ * @throws IOException
+ * Upon IO errors.
+ * <p>
+ * If an exception is thrown, the TFile will be in an inconsistent state. The only legitimate call after that would be close
+ */
+ public void append(byte[] key, int koff, int klen, byte[] value, int voff, int vlen) throws IOException {
+ if ((koff | klen | (koff + klen) | (key.length - (koff + klen))) < 0) {
+ throw new IndexOutOfBoundsException("Bad key buffer offset-length combination.");
+ }
+
+ if ((voff | vlen | (voff + vlen) | (value.length - (voff + vlen))) < 0) {
+ throw new IndexOutOfBoundsException("Bad value buffer offset-length combination.");
+ }
+
+ try {
+ DataOutputStream dosKey = prepareAppendKey(klen);
+ try {
+ ++errorCount;
+ dosKey.write(key, koff, klen);
+ --errorCount;
+ } finally {
+ dosKey.close();
+ }
+
+ DataOutputStream dosValue = prepareAppendValue(vlen);
+ try {
+ ++errorCount;
+ dosValue.write(value, voff, vlen);
+ --errorCount;
+ } finally {
+ dosValue.close();
+ }
+ } finally {
+ state = State.READY;
+ }
+ }
+
+ /**
+ * Helper class to register key after close call on key append stream.
+ */
+ private class KeyRegister extends DataOutputStream {
+ private final int expectedLength;
+ private boolean closed = false;
+
+ public KeyRegister(int len) {
+ super(currentKeyBufferOS);
+ if (len >= 0) {
+ currentKeyBufferOS.reset(len);
+ } else {
+ currentKeyBufferOS.reset();
+ }
+ expectedLength = len;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed == true) {
+ return;
+ }
+
+ try {
+ ++errorCount;
+ byte[] key = currentKeyBufferOS.getBuffer();
+ int len = currentKeyBufferOS.size();
+ /**
+ * verify length.
+ */
+ if (expectedLength >= 0 && expectedLength != len) {
+ throw new IOException("Incorrect key length: expected=" + expectedLength + " actual=" + len);
+ }
+
+ Utils.writeVInt(blkAppender, len);
+ blkAppender.write(key, 0, len);
+ if (tfileIndex.getFirstKey() == null) {
+ tfileIndex.setFirstKey(key, 0, len);
+ }
+
+ if (tfileMeta.isSorted()) {
+ byte[] lastKey = lastKeyBufferOS.getBuffer();
+ int lastLen = lastKeyBufferOS.size();
+ if (tfileMeta.getComparator().compare(key, 0, len, lastKey, 0, lastLen) < 0) {
+ throw new IOException("Keys are not added in sorted order");
+ }
+ }
+
+ BoundedByteArrayOutputStream tmp = currentKeyBufferOS;
+ currentKeyBufferOS = lastKeyBufferOS;
+ lastKeyBufferOS = tmp;
+ --errorCount;
+ } finally {
+ closed = true;
+ state = State.END_KEY;
+ }
+ }
+ }
+
+ /**
+ * Helper class to register value after close call on value append stream.
+ */
+ private class ValueRegister extends DataOutputStream {
+ private boolean closed = false;
+
+ public ValueRegister(OutputStream os) {
+ super(os);
+ }
+
+ // Avoiding flushing call to down stream.
+ @Override
+ public void flush() {
+ // do nothing
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed == true) {
+ return;
+ }
+
+ try {
+ ++errorCount;
+ super.close();
+ blkRecordCount++;
+ // bump up the total record count in the whole file
+ tfileMeta.incRecordCount();
+ finishDataBlock(false);
+ --errorCount;
+ } finally {
+ closed = true;
+ state = State.READY;
+ }
+ }
+ }
+
+ /**
+ * Obtain an output stream for writing a key into TFile. This may only be called when there is no active Key appending stream or value appending stream.
+ *
+ * @param length
+ * The expected length of the key. If length of the key is not known, set length = -1. Otherwise, the application must write exactly as many bytes
+ * as specified here before calling close on the returned output stream.
+ * @return The key appending output stream.
- * @throws IOException
+ *
+ */
+ public DataOutputStream prepareAppendKey(int length) throws IOException {
+ if (state != State.READY) {
+ throw new IllegalStateException("Incorrect state to start a new key: " + state.name());
+ }
+
+ initDataBlock();
+ DataOutputStream ret = new KeyRegister(length);
+ state = State.IN_KEY;
+ return ret;
+ }
+
+ /**
+ * Obtain an output stream for writing a value into TFile. This may only be called right after a key appending operation (the key append stream must be
+ * closed).
+ *
+ * @param length
+ * The expected length of the value. If length of the value is not known, set length = -1. Otherwise, the application must write exactly as many
+ * bytes as specified here before calling close on the returned output stream. Advertising the value size up-front guarantees that the value is
+ * encoded in one chunk, and avoids intermediate chunk buffering.
- * @throws IOException
- *
+ */
+ public DataOutputStream prepareAppendValue(int length) throws IOException {
+ if (state != State.END_KEY) {
+ throw new IllegalStateException("Incorrect state to start a new value: " + state.name());
+ }
+
+ DataOutputStream ret;
+
+ // unknown length
+ if (length < 0) {
+ if (valueBuffer == null) {
+ valueBuffer = new byte[getChunkBufferSize(conf)];
+ }
+ ret = new ValueRegister(new ChunkEncoder(blkAppender, valueBuffer));
+ } else {
+ ret = new ValueRegister(new Chunk.SingleChunkEncoder(blkAppender, length));
+ }
+
+ state = State.IN_VALUE;
+ return ret;
+ }
+
+ /**
+ * Obtain an output stream for creating a meta block. This function may not be called when there is a key append stream or value append stream active. No
+ * more key-value insertion is allowed after a meta data block has been added to TFile.
+ *
+ * @param name
+ * Name of the meta block.
+ * @param compressName
+ * Name of the compression algorithm to be used. Must be one of the strings returned by {@link TFile#getSupportedCompressionAlgorithms()}.
+ * @return A DataOutputStream that can be used to write Meta Block data. Closing the stream would signal the ending of the block.
- * @throws IOException
+ * @throws MetaBlockAlreadyExists
+ * the Meta Block with the same name already exists.
+ */
+ public DataOutputStream prepareMetaBlock(String name, String compressName) throws IOException, MetaBlockAlreadyExists {
+ if (state != State.READY) {
+ throw new IllegalStateException("Incorrect state to start a Meta Block: " + state.name());
+ }
+
+ finishDataBlock(true);
+ DataOutputStream outputStream = writerBCF.prepareMetaBlock(name, compressName);
+ return outputStream;
+ }
+
+ /**
+ * Obtain an output stream for creating a meta block. This function may not be called when there is a key append stream or value append stream active. No
+ * more key-value insertion is allowed after a meta data block has been added to TFile. Data will be compressed using the default compressor as defined in
+ * Writer's constructor.
+ *
+ * @param name
+ * Name of the meta block.
+ * @return A DataOutputStream that can be used to write Meta Block data. Closing the stream would signal the ending of the block.
- * @throws IOException
+ * @throws MetaBlockAlreadyExists
+ * the Meta Block with the same name already exists.
+ */
+ public DataOutputStream prepareMetaBlock(String name) throws IOException, MetaBlockAlreadyExists {
+ if (state != State.READY) {
+ throw new IllegalStateException("Incorrect state to start a Meta Block: " + state.name());
+ }
+
+ finishDataBlock(true);
+ return writerBCF.prepareMetaBlock(name);
+ }
+
+ /**
+ * Check if we need to start a new data block.
+ *
+ * @throws IOException
+ */
+ private void initDataBlock() throws IOException {
+ // for each new block, get a new appender
+ if (blkAppender == null) {
+ blkAppender = writerBCF.prepareDataBlock();
+ }
+ }
+
+ /**
+ * Close the current data block if necessary.
+ *
+ * @param bForceFinish
+ * Force the closure regardless of the block size.
+ * @throws IOException
+ */
+ void finishDataBlock(boolean bForceFinish) throws IOException {
+ if (blkAppender == null) {
+ return;
+ }
+
+ // exceeded the size limit, do the compression and finish the block
+ if (bForceFinish || blkAppender.getCompressedSize() >= sizeMinBlock) {
+ // keep tracks of the last key of each data block, no padding
+ // for now
+ TFileIndexEntry keyLast = new TFileIndexEntry(lastKeyBufferOS.getBuffer(), 0, lastKeyBufferOS.size(), blkRecordCount);
+ tfileIndex.addEntry(keyLast);
+ // close the appender
+ blkAppender.close();
+ blkAppender = null;
+ blkRecordCount = 0;
+ }
+ }
+ }
+
+ /**
+ * TFile Reader. Users may only read TFiles by creating TFile.Reader.Scanner. objects. A scanner may scan the whole TFile ({@link Reader#createScanner()} ) ,
+ * a portion of TFile based on byte offsets ( {@link Reader#createScanner(long, long)}), or a portion of TFile with keys fall in a certain key range (for
+ * sorted TFile only, {@link Reader#createScanner(byte[], byte[])} or {@link Reader#createScanner(RawComparable, RawComparable)}).
+ */
+ public static class Reader implements Closeable {
+ // The underlying BCFile reader.
+ final BCFile.Reader readerBCF;
+
+ // TFile index, it is loaded lazily.
+ TFileIndex tfileIndex = null;
+ final TFileMeta tfileMeta;
+ final BytesComparator comparator;
+
+ // global begin and end locations.
+ private final Location begin;
+ private final Location end;
+
+ /**
+ * Location representing a virtual position in the TFile.
+ */
+ static final class Location implements Comparable<Location>, Cloneable {
+ private int blockIndex;
+ // distance/offset from the beginning of the block
+ private long recordIndex;
+
+ Location(int blockIndex, long recordIndex) {
+ set(blockIndex, recordIndex);
+ }
+
+ void incRecordIndex() {
+ ++recordIndex;
+ }
+
+ Location(Location other) {
+ set(other);
+ }
+
+ int getBlockIndex() {
+ return blockIndex;
+ }
+
+ long getRecordIndex() {
+ return recordIndex;
+ }
+
+ void set(int blockIndex, long recordIndex) {
+ if ((blockIndex | recordIndex) < 0) {
+ throw new IllegalArgumentException("Illegal parameter for BlockLocation.");
+ }
+ this.blockIndex = blockIndex;
+ this.recordIndex = recordIndex;
+ }
+
+ void set(Location other) {
+ set(other.blockIndex, other.recordIndex);
+ }
+
+ /**
+ * @see java.lang.Comparable#compareTo(java.lang.Object)
+ */
+ @Override
+ public int compareTo(Location other) {
+ return compareTo(other.blockIndex, other.recordIndex);
+ }
+
+ int compareTo(int bid, long rid) {
+ if (this.blockIndex == bid) {
+ long ret = this.recordIndex - rid;
+ if (ret > 0)
+ return 1;
+ if (ret < 0)
+ return -1;
+ return 0;
+ }
+ return this.blockIndex - bid;
+ }
+
+ /**
+ * @see java.lang.Object#clone()
+ */
+ @Override
+ protected Location clone() {
+ return new Location(blockIndex, recordIndex);
+ }
+
+ /**
+ * @see java.lang.Object#hashCode()
+ */
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = prime + blockIndex;
+ result = (int) (prime * result + recordIndex);
+ return result;
+ }
+
+ /**
+ * @see java.lang.Object#equals(java.lang.Object)
+ */
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ Location other = (Location) obj;
+ if (blockIndex != other.blockIndex)
+ return false;
+ if (recordIndex != other.recordIndex)
+ return false;
+ return true;
+ }
+ }
+
+ /**
+ * Constructor
+ *
+ * @param fsdis
+ * FS input stream of the TFile.
+ * @param fileLength
+ * The length of TFile. This is required because we have no easy way of knowing the actual size of the input file through the File input stream.
- * @param conf
- * @throws IOException
+ */
+ public Reader(FSDataInputStream fsdis, long fileLength, Configuration conf) throws IOException {
+ readerBCF = new BCFile.Reader(fsdis, fileLength, conf);
+
+ // first, read TFile meta
+ BlockReader brMeta = readerBCF.getMetaBlock(TFileMeta.BLOCK_NAME);
+ try {
+ tfileMeta = new TFileMeta(brMeta);
+ } finally {
+ brMeta.close();
+ }
+
+ comparator = tfileMeta.getComparator();
+ // Set begin and end locations.
+ begin = new Location(0, 0);
+ end = new Location(readerBCF.getBlockCount(), 0);
+ }
+
+ /**
+ * Close the reader. The state of the Reader object is undefined after close. Calling close() for multiple times has no effect.
+ */
+ public void close() throws IOException {
+ readerBCF.close();
+ }
+
+ /**
+ * Get the begin location of the TFile.
+ *
+ * @return If TFile is not empty, the location of the first key-value pair. Otherwise, it returns end().
+ */
+ Location begin() {
+ return begin;
+ }
+
+ /**
+ * Get the end location of the TFile.
+ *
+ * @return The location right after the last key-value pair in TFile.
+ */
+ Location end() {
+ return end;
+ }
+
+ /**
+ * Get the string representation of the comparator.
+ *
+ * @return If the TFile is not sorted by keys, an empty string will be returned. Otherwise, the actual comparator string that is provided during the TFile
+ * creation time will be returned.
+ */
+ public String getComparatorName() {
+ return tfileMeta.getComparatorString();
+ }
+
+ /**
+ * Is the TFile sorted?
+ *
+ * @return true if TFile is sorted.
+ */
+ public boolean isSorted() {
+ return tfileMeta.isSorted();
+ }
+
+ /**
+ * Get the number of key-value pair entries in TFile.
+ *
+ * @return the number of key-value pairs in TFile
+ */
+ public long getEntryCount() {
+ return tfileMeta.getRecordCount();
+ }
+
+ /**
+ * Lazily loading the TFile index.
+ *
+ * @throws IOException
+ */
+ synchronized void checkTFileDataIndex() throws IOException {
+ if (tfileIndex == null) {
+ BlockReader brIndex = readerBCF.getMetaBlock(TFileIndex.BLOCK_NAME);
+ try {
+ tfileIndex = new TFileIndex(readerBCF.getBlockCount(), brIndex, tfileMeta.getComparator());
+ } finally {
+ brIndex.close();
+ }
+ }
+ }
+
+ /**
+ * Get the first key in the TFile.
+ *
+ * @return The first key in the TFile.
- * @throws IOException
+ */
+ public RawComparable getFirstKey() throws IOException {
+ checkTFileDataIndex();
+ return tfileIndex.getFirstKey();
+ }
+
+ /**
+ * Get the last key in the TFile.
+ *
+ * @return The last key in the TFile.
- * @throws IOException
+ */
+ public RawComparable getLastKey() throws IOException {
+ checkTFileDataIndex();
+ return tfileIndex.getLastKey();
+ }
+
+ /**
+ * Get a Comparator object to compare Entries. It is useful when you want stores the entries in a collection (such as PriorityQueue) and perform sorting or
+ * comparison among entries based on the keys without copying out the key.
+ *
+ * @return An Entry Comparator..
+ */
+ public Comparator<Scanner.Entry> getEntryComparator() {
+ if (!isSorted()) {
+ throw new RuntimeException("Entries are not comparable for unsorted TFiles");
+ }
+
+ return new Comparator<Scanner.Entry>() {
+ /**
+ * Provide a customized comparator for Entries. This is useful if we have a collection of Entry objects. However, if the Entry objects come from
+ * different TFiles, users must ensure that those TFiles share the same RawComparator.
+ */
+ @Override
+ public int compare(Scanner.Entry o1, Scanner.Entry o2) {
+ return comparator.compare(o1.getKeyBuffer(), 0, o1.getKeyLength(), o2.getKeyBuffer(), 0, o2.getKeyLength());
+ }
+ };
+ }
+
+ /**
+ * Get an instance of the RawComparator that is constructed based on the string comparator representation.
+ *
+ * @return a Comparator that can compare RawComparable's.
+ */
+ public Comparator<RawComparable> getComparator() {
+ return comparator;
+ }
+
+ /**
+ * Stream access to a meta block.``
+ *
+ * @param name
+ * The name of the meta block.
+ * @return The input stream.
+ * @throws IOException
+ * on I/O error.
+ * @throws MetaBlockDoesNotExist
+ * If the meta block with the name does not exist.
+ */
+ public DataInputStream getMetaBlock(String name) throws IOException, MetaBlockDoesNotExist {
+ return readerBCF.getMetaBlock(name);
+ }
+
+ /**
+ * if greater is true then returns the beginning location of the block containing the key strictly greater than input key. if greater is false then returns
+ * the beginning location of the block greater than equal to the input key
+ *
+ * @param key
+ * the input key
+ * @param greater
+ * boolean flag
+ * @throws IOException
+ */
+ Location getBlockContainsKey(RawComparable key, boolean greater) throws IOException {
+ if (!isSorted()) {
+ throw new RuntimeException("Seeking in unsorted TFile");
+ }
+ checkTFileDataIndex();
+ int blkIndex = (greater) ? tfileIndex.upperBound(key) : tfileIndex.lowerBound(key);
+ if (blkIndex < 0)
+ return end;
+ return new Location(blkIndex, 0);
+ }
+
+ int compareKeys(byte[] a, int o1, int l1, byte[] b, int o2, int l2) {
+ if (!isSorted()) {
+ throw new RuntimeException("Cannot compare keys for unsorted TFiles.");
+ }
+ return comparator.compare(a, o1, l1, b, o2, l2);
+ }
+
+ int compareKeys(RawComparable a, RawComparable b) {
+ if (!isSorted()) {
+ throw new RuntimeException("Cannot compare keys for unsorted TFiles.");
+ }
+ return comparator.compare(a, b);
+ }
+
+ /**
+ * Get the location pointing to the beginning of the first key-value pair in a compressed block whose byte offset in the TFile is greater than or equal to
+ * the specified offset.
+ *
+ * @param offset
+ * the user supplied offset.
+ * @return the location to the corresponding entry; or end() if no such entry exists.
+ */
+ Location getLocationNear(long offset) {
+ int blockIndex = readerBCF.getBlockIndexNear(offset);
+ if (blockIndex == -1)
+ return end;
+ return new Location(blockIndex, 0);
+ }
+
+ /**
+ * Get a sample key that is within a block whose starting offset is greater than or equal to the specified offset.
+ *
+ * @param offset
+ * The file offset.
+ * @return the key that fits the requirement; or null if no such key exists (which could happen if the offset is close to the end of the TFile).
- * @throws IOException
+ */
+ public RawComparable getKeyNear(long offset) throws IOException {
+ int blockIndex = readerBCF.getBlockIndexNear(offset);
+ if (blockIndex == -1)
+ return null;
+ checkTFileDataIndex();
+ return new ByteArray(tfileIndex.getEntry(blockIndex).key);
+ }
+
+ /**
+ * Get a scanner than can scan the whole TFile.
+ *
+ * @return The scanner object. A valid Scanner is always returned even if the TFile is empty.
- * @throws IOException
+ */
+ public Scanner createScanner() throws IOException {
+ return new Scanner(this, begin, end);
+ }
+
+ /**
+ * Get a scanner that covers a portion of TFile based on byte offsets.
+ *
+ * @param offset
+ * The beginning byte offset in the TFile.
+ * @param length
+ * The length of the region.
+ * @return The actual coverage of the returned scanner tries to match the specified byte-region but always round up to the compression block boundaries. It
+ * is possible that the returned scanner contains zero key-value pairs even if length is positive.
- * @throws IOException
+ */
+ public Scanner createScanner(long offset, long length) throws IOException {
+ return new Scanner(this, offset, offset + length);
+ }
+
+ /**
+ * Get a scanner that covers a portion of TFile based on keys.
+ *
+ * @param beginKey
+ * Begin key of the scan (inclusive). If null, scan from the first key-value entry of the TFile.
+ * @param endKey
+ * End key of the scan (exclusive). If null, scan up to the last key-value entry of the TFile.
+ * @return The actual coverage of the returned scanner will cover all keys greater than or equal to the beginKey and less than the endKey.
- * @throws IOException
+ */
+ public Scanner createScanner(byte[] beginKey, byte[] endKey) throws IOException {
+ return createScanner((beginKey == null) ? null : new ByteArray(beginKey, 0, beginKey.length), (endKey == null) ? null : new ByteArray(endKey, 0,
+ endKey.length));
+ }
+
+ /**
+ * Get a scanner that covers a specific key range.
+ *
+ * @param beginKey
+ * Begin key of the scan (inclusive). If null, scan from the first key-value entry of the TFile.
+ * @param endKey
+ * End key of the scan (exclusive). If null, scan up to the last key-value entry of the TFile.
+ * @return The actual coverage of the returned scanner will cover all keys greater than or equal to the beginKey and less than the endKey.
- * @throws IOException
+ */
+ public Scanner createScanner(RawComparable beginKey, RawComparable endKey) throws IOException {
+ if ((beginKey != null) && (endKey != null) && (compareKeys(beginKey, endKey) >= 0)) {
+ return new Scanner(this, beginKey, beginKey);
+ }
+ return new Scanner(this, beginKey, endKey);
+ }
+
+ /**
+ * The TFile Scanner. The Scanner has an implicit cursor, which, upon creation, points to the first key-value pair in the scan range. If the scan range is
+ * empty, the cursor will point to the end of the scan range.
+ * <p>
+ * Use {@link Scanner#atEnd()} to test whether the cursor is at the end location of the scanner.
+ * <p>
+ * Use {@link Scanner#advance()} to move the cursor to the next key-value pair (or end if none exists). Use seekTo methods ( {@link Scanner#seekTo(byte[])}
+ * or {@link Scanner#seekTo(byte[], int, int)}) to seek to any arbitrary location in the covered range (including backward seeking). Use
+ * {@link Scanner#rewind()} to seek back to the beginning of the scanner. Use {@link Scanner#seekToEnd()} to seek to the end of the scanner.
+ * <p>
+ * Actual keys and values may be obtained through {@link Scanner.Entry} object, which is obtained through {@link Scanner#entry()}.
+ */
+ public static class Scanner implements Closeable {
+ // The underlying TFile reader.
+ final Reader reader;
+ // current block (null if reaching end)
+ private BlockReader blkReader;
+
+ Location beginLocation;
+ Location endLocation;
+ Location currentLocation;
+
+ // flag to ensure value is only examined once.
+ boolean valueChecked = false;
+ // reusable buffer for keys.
+ final byte[] keyBuffer;
+ // length of key, -1 means key is invalid.
+ int klen = -1;
+
+ static final int MAX_VAL_TRANSFER_BUF_SIZE = 128 * 1024;
+ BytesWritable valTransferBuffer;
+
+ DataInputBuffer keyDataInputStream;
+ ChunkDecoder valueBufferInputStream;
+ DataInputStream valueDataInputStream;
+ // vlen == -1 if unknown.
+ int vlen;
+
+ /**
+ * Constructor
+ *
+ * @param reader
+ * The TFile reader object.
+ * @param offBegin
+ * Begin byte-offset of the scan.
+ * @param offEnd
+ * End byte-offset of the scan.
+ * @throws IOException
+ *
+ * The offsets will be rounded to the beginning of a compressed block whose offset is greater than or equal to the specified offset.
+ */
+ protected Scanner(Reader reader, long offBegin, long offEnd) throws IOException {
+ this(reader, reader.getLocationNear(offBegin), reader.getLocationNear(offEnd));
+ }
+
+ /**
+ * Constructor
+ *
+ * @param reader
+ * The TFile reader object.
+ * @param begin
+ * Begin location of the scan.
+ * @param end
+ * End location of the scan.
+ * @throws IOException
+ */
+ Scanner(Reader reader, Location begin, Location end) throws IOException {
+ this.reader = reader;
+ // ensure the TFile index is loaded throughout the life of scanner.
+ reader.checkTFileDataIndex();
+ beginLocation = begin;
+ endLocation = end;
+
+ valTransferBuffer = new BytesWritable();
+ keyBuffer = new byte[MAX_KEY_SIZE];
+ keyDataInputStream = new DataInputBuffer();
+ valueBufferInputStream = new ChunkDecoder();
+ valueDataInputStream = new DataInputStream(valueBufferInputStream);
+
+ if (beginLocation.compareTo(endLocation) >= 0) {
+ currentLocation = new Location(endLocation);
+ } else {
+ currentLocation = new Location(0, 0);
+ initBlock(beginLocation.getBlockIndex());
+ inBlockAdvance(beginLocation.getRecordIndex());
+ }
+ }
+
+ /**
+ * Constructor
+ *
+ * @param reader
+ * The TFile reader object.
+ * @param beginKey
+ * Begin key of the scan. If null, scan from the first <K,V> entry of the TFile.
+ * @param endKey
+ * End key of the scan. If null, scan up to the last <K, V> entry of the TFile.
- * @throws IOException
+ */
+ protected Scanner(Reader reader, RawComparable beginKey, RawComparable endKey) throws IOException {
+ this(reader, (beginKey == null) ? reader.begin() : reader.getBlockContainsKey(beginKey, false), reader.end());
+ if (beginKey != null) {
+ inBlockAdvance(beginKey, false);
+ beginLocation.set(currentLocation);
+ }
+ if (endKey != null) {
+ seekTo(endKey, false);
+ endLocation.set(currentLocation);
+ seekTo(beginLocation);
+ }
+ }
+
+ /**
+ * Move the cursor to the first entry whose key is greater than or equal to the input key. Synonymous to seekTo(key, 0, key.length). The entry returned by
+ * the previous entry() call will be invalid.
+ *
+ * @param key
+ * The input key
+ * @return true if we find an equal key.
- * @throws IOException
+ */
+ public boolean seekTo(byte[] key) throws IOException {
+ return seekTo(key, 0, key.length);
+ }
+
+ /**
+ * Move the cursor to the first entry whose key is greater than or equal to the input key. The entry returned by the previous entry() call will be
+ * invalid.
+ *
+ * @param key
+ * The input key
+ * @param keyOffset
+ * offset in the key buffer.
+ * @param keyLen
+ * key buffer length.
+ * @return true if we find an equal key; false otherwise.
- * @throws IOException
+ */
+ public boolean seekTo(byte[] key, int keyOffset, int keyLen) throws IOException {
+ return seekTo(new ByteArray(key, keyOffset, keyLen), false);
+ }
+
+ private boolean seekTo(RawComparable key, boolean beyond) throws IOException {
+ Location l = reader.getBlockContainsKey(key, beyond);
+ if (l.compareTo(beginLocation) < 0) {
+ l = beginLocation;
+ } else if (l.compareTo(endLocation) >= 0) {
+ seekTo(endLocation);
+ return false;
+ }
+
+ // check if what we are seeking is in the later part of the current
+ // block.
+ if (atEnd() || (l.getBlockIndex() != currentLocation.getBlockIndex()) || (compareCursorKeyTo(key) >= 0)) {
+ // sorry, we must seek to a different location first.
+ seekTo(l);
+ }
+
+ return inBlockAdvance(key, beyond);
+ }
+
+ /**
+ * Move the cursor to the new location. The entry returned by the previous entry() call will be invalid.
+ *
+ * @param l
+ * new cursor location. It must fall between the begin and end location of the scanner.
+ * @throws IOException
+ */
+ private void seekTo(Location l) throws IOException {
+ if (l.compareTo(beginLocation) < 0) {
+ throw new IllegalArgumentException("Attempt to seek before the begin location.");
+ }
+
+ if (l.compareTo(endLocation) > 0) {
+ throw new IllegalArgumentException("Attempt to seek after the end location.");
+ }
+
+ if (l.compareTo(endLocation) == 0) {
+ parkCursorAtEnd();
+ return;
+ }
+
+ if (l.getBlockIndex() != currentLocation.getBlockIndex()) {
+ // going to a totally different block
+ initBlock(l.getBlockIndex());
+ } else {
+ if (valueChecked) {
+ // may temporarily go beyond the last record in the block (in which
+ // case the next if loop will always be true).
+ inBlockAdvance(1);
+ }
+ if (l.getRecordIndex() < currentLocation.getRecordIndex()) {
+ initBlock(l.getBlockIndex());
+ }
+ }
+
+ inBlockAdvance(l.getRecordIndex() - currentLocation.getRecordIndex());
+
+ return;
+ }
+
+ /**
+ * Rewind to the first entry in the scanner. The entry returned by the previous entry() call will be invalid.
- *
- * @throws IOException
+ */
+ public void rewind() throws IOException {
+ seekTo(beginLocation);
+ }
+
+ /**
+ * Seek to the end of the scanner. The entry returned by the previous entry() call will be invalid.
- *
- * @throws IOException
+ */
+ public void seekToEnd() throws IOException {
+ parkCursorAtEnd();
+ }
+
+ /**
+ * Move the cursor to the first entry whose key is greater than or equal to the input key. Synonymous to lowerBound(key, 0, key.length). The entry
+ * returned by the previous entry() call will be invalid.
+ *
+ * @param key
+ * The input key
- * @throws IOException
+ */
+ public void lowerBound(byte[] key) throws IOException {
+ lowerBound(key, 0, key.length);
+ }
+
+ /**
+ * Move the cursor to the first entry whose key is greater than or equal to the input key. The entry returned by the previous entry() call will be
+ * invalid.
+ *
+ * @param key
+ * The input key
+ * @param keyOffset
+ * offset in the key buffer.
+ * @param keyLen
+ * key buffer length.
- * @throws IOException
+ */
+ public void lowerBound(byte[] key, int keyOffset, int keyLen) throws IOException {
+ seekTo(new ByteArray(key, keyOffset, keyLen), false);
+ }
+
+ /**
+ * Move the cursor to the first entry whose key is strictly greater than the input key. Synonymous to upperBound(key, 0, key.length). The entry returned
+ * by the previous entry() call will be invalid.
+ *
+ * @param key
+ * The input key
- * @throws IOException
+ */
+ public void upperBound(byte[] key) throws IOException {
+ upperBound(key, 0, key.length);
+ }
+
+ /**
+ * Move the cursor to the first entry whose key is strictly greater than the input key. The entry returned by the previous entry() call will be invalid.
+ *
+ * @param key
+ * The input key
+ * @param keyOffset
+ * offset in the key buffer.
+ * @param keyLen
+ * key buffer length.
- * @throws IOException
+ */
+ public void upperBound(byte[] key, int keyOffset, int keyLen) throws IOException {
+ seekTo(new ByteArray(key, keyOffset, keyLen), true);
+ }
+
+ /**
+ * Move the cursor to the next key-value pair. The entry returned by the previous entry() call will be invalid.
+ *
+ * @return true if the cursor successfully moves. False when cursor is already at the end location and cannot be advanced.
- * @throws IOException
+ */
+ public boolean advance() throws IOException {
+ if (atEnd()) {
+ return false;
+ }
+
+ int curBid = currentLocation.getBlockIndex();
+ long curRid = currentLocation.getRecordIndex();
+ long entriesInBlock = reader.getBlockEntryCount(curBid);
+ if (curRid + 1 >= entriesInBlock) {
+ if (endLocation.compareTo(curBid + 1, 0) <= 0) {
+ // last entry in TFile.
+ parkCursorAtEnd();
+ } else {
+ // last entry in Block.
+ initBlock(curBid + 1);
+ }
+ } else {
+ inBlockAdvance(1);
+ }
+ return true;
+ }
+
+ /**
+ * Load a compressed block for reading. Expecting blockIndex is valid.
+ *
+ * @throws IOException
+ */
+ private void initBlock(int blockIndex) throws IOException {
+ klen = -1;
+ if (blkReader != null) {
+ try {
+ blkReader.close();
+ } finally {
+ blkReader = null;
+ }
+ }
+ blkReader = reader.getBlockReader(blockIndex);
+ currentLocation.set(blockIndex, 0);
+ }
+
+ private void parkCursorAtEnd() throws IOException {
+ klen = -1;
+ currentLocation.set(endLocation);
+ if (blkReader != null) {
+ try {
+ blkReader.close();
+ } finally {
+ blkReader = null;
+ }
+ }
+ }
+
+ /**
+ * Close the scanner. Release all resources. The behavior of using the scanner after calling close is not defined. The entry returned by the previous
+ * entry() call will be invalid.
+ */
+ public void close() throws IOException {
+ parkCursorAtEnd();
+ }
+
+ /**
+ * Is cursor at the end location?
+ *
+ * @return true if the cursor is at the end location.
+ */
+ public boolean atEnd() {
+ return (currentLocation.compareTo(endLocation) >= 0);
+ }
+
+ /**
+ * check whether we have already successfully obtained the key. It also initializes the valueInputStream.
+ */
+ void checkKey() throws IOException {
+ if (klen >= 0)
+ return;
+ if (atEnd()) {
+ throw new EOFException("No key-value to read");
+ }
+ klen = -1;
+ vlen = -1;
+ valueChecked = false;
+
+ klen = Utils.readVInt(blkReader);
+ blkReader.readFully(keyBuffer, 0, klen);
+ valueBufferInputStream.reset(blkReader);
+ if (valueBufferInputStream.isLastChunk()) {
+ vlen = valueBufferInputStream.getRemain();
+ }
+ }
+
+ /**
+ * Get an entry to access the key and value.
+ *
+ * @return The Entry object to access the key and value.
- * @throws IOException
+ */
+ public Entry entry() throws IOException {
+ checkKey();
+ return new Entry();
+ }
+
+ /**
+ * Internal API. Comparing the key at cursor to user-specified key.
+ *
+ * @param other
+ * user-specified key.
+ * @return negative if key at cursor is smaller than user key; 0 if equal; and positive if key at cursor greater than user key.
+ * @throws IOException
+ */
+ int compareCursorKeyTo(RawComparable other) throws IOException {
+ checkKey();
+ return reader.compareKeys(keyBuffer, 0, klen, other.buffer(), other.offset(), other.size());
+ }
+
+ /**
+ * Entry to a <Key, Value> pair.
+ */
+ public class Entry implements Comparable<RawComparable> {
+ /**
+ * Get the length of the key.
+ *
+ * @return the length of the key.
+ */
+ public int getKeyLength() {
+ return klen;
+ }
+
+ byte[] getKeyBuffer() {
+ return keyBuffer;
+ }
+
+ /**
+ * Copy the key and value in one shot into BytesWritables. This is equivalent to getKey(key); getValue(value);
+ *
+ * @param key
+ * BytesWritable to hold key.
+ * @param value
+ * BytesWritable to hold value
- * @throws IOException
+ */
+ public void get(BytesWritable key, BytesWritable value) throws IOException {
+ getKey(key);
+ getValue(value);
+ }
+
+ /**
+ * Copy the key into BytesWritable. The input BytesWritable will be automatically resized to the actual key size.
+ *
+ * @param key
+ * BytesWritable to hold the key.
- * @throws IOException
+ */
+ public int getKey(BytesWritable key) throws IOException {
+ key.setSize(getKeyLength());
+ getKey(key.getBytes());
+ return key.getLength();
+ }
+
+ /**
+ * Copy the value into BytesWritable. The input BytesWritable will be automatically resized to the actual value size. The implementation directly uses
+ * the buffer inside BytesWritable for storing the value. The call does not require the value length to be known.
- *
- * @param value
- * @throws IOException
+ */
+ public long getValue(BytesWritable value) throws IOException {
+ DataInputStream dis = getValueStream();
+ int size = 0;
+ try {
+ int remain;
+ while ((remain = valueBufferInputStream.getRemain()) > 0) {
+ value.setSize(size + remain);
+ dis.readFully(value.getBytes(), size, remain);
+ size += remain;
+ }
+ return value.getLength();
+ } finally {
+ dis.close();
+ }
+ }
+
+ /**
+ * Writing the key to the output stream. This method avoids copying key buffer from Scanner into user buffer, then writing to the output stream.
+ *
+ * @param out
+ * The output stream
+ * @return the length of the key.
- * @throws IOException
+ */
+ public int writeKey(OutputStream out) throws IOException {
+ out.write(keyBuffer, 0, klen);
+ return klen;
+ }
+
+ /**
+ * Writing the value to the output stream. This method avoids copying value data from Scanner into user buffer, then writing to the output stream. It
+ * does not require the value length to be known.
+ *
+ * @param out
+ * The output stream
+ * @return the length of the value
- * @throws IOException
+ */
+ public long writeValue(OutputStream out) throws IOException {
+ DataInputStream dis = getValueStream();
+ long size = 0;
+ try {
+ int chunkSize;
+ while ((chunkSize = valueBufferInputStream.getRemain()) > 0) {
+ chunkSize = Math.min(chunkSize, MAX_VAL_TRANSFER_BUF_SIZE);
+ valTransferBuffer.setSize(chunkSize);
+ dis.readFully(valTransferBuffer.getBytes(), 0, chunkSize);
+ out.write(valTransferBuffer.getBytes(), 0, chunkSize);
+ size += chunkSize;
+ }
+ return size;
+ } finally {
+ dis.close();
+ }
+ }
+
+ /**
+ * Copy the key into user supplied buffer.
+ *
+ * @param buf
+ * The buffer supplied by user. The length of the buffer must not be shorter than the key length.
+ * @return The length of the key.
- *
- * @throws IOException
+ */
+ public int getKey(byte[] buf) throws IOException {
+ return getKey(buf, 0);
+ }
+
+ /**
+ * Copy the key into user supplied buffer.
+ *
+ * @param buf
+ * The buffer supplied by user.
+ * @param offset
+ * The starting offset of the user buffer where we should copy the key into. Requiring the key-length + offset no greater than the buffer
+ * length.
+ * @return The length of the key.
- * @throws IOException
+ */
+ public int getKey(byte[] buf, int offset) throws IOException {
+ if ((offset | (buf.length - offset - klen)) < 0) {
+ throw new IndexOutOfBoundsException("Bufer not enough to store the key");
+ }
+ System.arraycopy(keyBuffer, 0, buf, offset, klen);
+ return klen;
+ }
+
+ /**
+ * Streaming access to the key. Useful for desrializing the key into user objects.
+ *
+ * @return The input stream.
+ */
+ public DataInputStream getKeyStream() {
+ keyDataInputStream.reset(keyBuffer, klen);
+ return keyDataInputStream;
+ }
+
+ /**
+ * Get the length of the value. isValueLengthKnown() must be tested true.
+ *
+ * @return the length of the value.
+ */
+ public int getValueLength() {
+ if (vlen >= 0) {
+ return vlen;
+ }
+
+ throw new RuntimeException("Value length unknown.");
+ }
+
+ /**
+ * Copy value into user-supplied buffer. User supplied buffer must be large enough to hold the whole value. The value part of the key-value pair pointed
+ * by the current cursor is not cached and can only be examined once. Calling any of the following functions more than once without moving the cursor
+ * will result in exception: {@link #getValue(byte[])}, {@link #getValue(byte[], int)}, {@link #getValueStream}.
+ *
+ * @return the length of the value. Does not require isValueLengthKnown() to be true.
- * @throws IOException
+ *
+ */
+ public int getValue(byte[] buf) throws IOException {
+ return getValue(buf, 0);
+ }
+
+ /**
+ * Copy value into user-supplied buffer. User supplied buffer must be large enough to hold the whole value (starting from the offset). The value part of
+ * the key-value pair pointed by the current cursor is not cached and can only be examined once. Calling any of the following functions more than once
+ * without moving the cursor will result in exception: {@link #getValue(byte[])}, {@link #getValue(byte[], int)}, {@link #getValueStream}.
+ *
+ * @return the length of the value. Does not require isValueLengthKnown() to be true.
- * @throws IOException
+ */
+ public int getValue(byte[] buf, int offset) throws IOException {
+ DataInputStream dis = getValueStream();
+ try {
+ if (isValueLengthKnown()) {
+ if ((offset | (buf.length - offset - vlen)) < 0) {
+ throw new IndexOutOfBoundsException("Buffer too small to hold value");
+ }
+ dis.readFully(buf, offset, vlen);
+ return vlen;
+ }
+
+ int nextOffset = offset;
+ while (nextOffset < buf.length) {
+ int n = dis.read(buf, nextOffset, buf.length - nextOffset);
+ if (n < 0) {
+ break;
+ }
+ nextOffset += n;
+ }
+ if (dis.read() >= 0) {
+ // attempt to read one more byte to determine whether we reached
+ // the
+ // end or not.
+ throw new IndexOutOfBoundsException("Buffer too small to hold value");
+ }
+ return nextOffset - offset;
+ } finally {
+ dis.close();
+ }
+ }
+
+ /**
+ * Stream access to value. The value part of the key-value pair pointed by the current cursor is not cached and can only be examined once. Calling any
+ * of the following functions more than once without moving the cursor will result in exception: {@link #getValue(byte[])},
+ * {@link #getValue(byte[], int)}, {@link #getValueStream}.
+ *
+ * @return The input stream for reading the value.
- * @throws IOException
+ */
+ public DataInputStream getValueStream() throws IOException {
+ if (valueChecked == true) {
+ throw new IllegalStateException("Attempt to examine value multiple times.");
+ }
+ valueChecked = true;
+ return valueDataInputStream;
+ }
+
+ /**
+ * Check whether it is safe to call getValueLength().
+ *
+ * @return true if value length is known before hand. Values less than the chunk size will always have their lengths known before hand. Values that are
+ * written out as a whole (with advertised length up-front) will always have their lengths known in read.
+ */
+ public boolean isValueLengthKnown() {
+ return (vlen >= 0);
+ }
+
+ /**
+ * Compare the entry key to another key. Synonymous to compareTo(key, 0, key.length).
+ *
+ * @param buf
+ * The key buffer.
+ * @return comparison result between the entry key with the input key.
+ */
+ public int compareTo(byte[] buf) {
+ return compareTo(buf, 0, buf.length);
+ }
+
+ /**
+ * Compare the entry key to another key. Synonymous to compareTo(new ByteArray(buf, offset, length)
+ *
+ * @param buf
+ * The key buffer
+ * @param offset
+ * offset into the key buffer.
+ * @param length
+ * the length of the key.
+ * @return comparison result between the entry key with the input key.
+ */
+ public int compareTo(byte[] buf, int offset, int length) {
+ return compareTo(new ByteArray(buf, offset, length));
+ }
+
+ /**
+ * Compare an entry with a RawComparable object. This is useful when Entries are stored in a collection, and we want to compare a user supplied key.
+ */
+ @Override
+ public int compareTo(RawComparable key) {
+ return reader.compareKeys(keyBuffer, 0, getKeyLength(), key.buffer(), key.offset(), key.size());
+ }
+
+ /**
+ * Compare whether this and other points to the same key value.
+ */
+ @Override
+ public boolean equals(Object other) {
+ if (this == other)
+ return true;
+ if (!(other instanceof Entry))
+ return false;
+ return ((Entry) other).compareTo(keyBuffer, 0, getKeyLength()) == 0;
+ }
+
+ @Override
+ public int hashCode() {
+ return WritableComparator.hashBytes(keyBuffer, getKeyLength());
+ }
+ }
+
+ /**
+ * Advance cursor by n positions within the block.
+ *
+ * @param n
+ * Number of key-value pairs to skip in block.
+ * @throws IOException
+ */
+ private void inBlockAdvance(long n) throws IOException {
+ for (long i = 0; i < n; ++i) {
+ checkKey();
+ if (!valueBufferInputStream.isClosed()) {
+ valueBufferInputStream.close();
+ }
+ klen = -1;
+ currentLocation.incRecordIndex();
+ }
+ }
+
+ /**
+ * Advance cursor in block until we find a key that is greater than or equal to the input key.
+ *
+ * @param key
+ * Key to compare.
+ * @param greater
+ * advance until we find a key greater than the input key.
+ * @return true if we find a equal key.
+ * @throws IOException
+ */
+ private boolean inBlockAdvance(RawComparable key, boolean greater) throws IOException {
+ int curBid = currentLocation.getBlockIndex();
+ long entryInBlock = reader.getBlockEntryCount(curBid);
+ if (curBid == endLocation.getBlockIndex()) {
+ entryInBlock = endLocation.getRecordIndex();
+ }
+
+ while (currentLocation.getRecordIndex() < entryInBlock) {
+ int cmp = compareCursorKeyTo(key);
+ if (cmp > 0)
+ return false;
+ if (cmp == 0 && !greater)
+ return true;
+ if (!valueBufferInputStream.isClosed()) {
+ valueBufferInputStream.close();
+ }
+ klen = -1;
+ currentLocation.incRecordIndex();
+ }
+
+ throw new RuntimeException("Cannot find matching key in block.");
+ }
+ }
+
+ long getBlockEntryCount(int curBid) {
+ return tfileIndex.getEntry(curBid).entries();
+ }
+
+ BlockReader getBlockReader(int blockIndex) throws IOException {
+ return readerBCF.getDataBlock(blockIndex);
+ }
+ }
+
+ /**
+ * Data structure representing "TFile.meta" meta block.
+ */
+ static final class TFileMeta {
+ final static String BLOCK_NAME = "TFile.meta";
+ final Version version;
+ private long recordCount;
+ private final String strComparator;
+ private final BytesComparator comparator;
+
+ // ctor for writes
+ public TFileMeta(String comparator) {
+ // set fileVersion to API version when we create it.
+ version = TFile.API_VERSION;
+ recordCount = 0;
+ strComparator = (comparator == null) ? "" : comparator;
+ this.comparator = makeComparator(strComparator);
+ }
+
+ // ctor for reads
+ public TFileMeta(DataInput in) throws IOException {
+ version = new Version(in);
+ if (!version.compatibleWith(TFile.API_VERSION)) {
+ throw new RuntimeException("Incompatible TFile fileVersion.");
+ }
+ recordCount = Utils.readVLong(in);
+ strComparator = Utils.readString(in);
+ comparator = makeComparator(strComparator);
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ static BytesComparator makeComparator(String comparator) {
+ if (comparator.length() == 0) {
+ // unsorted keys
+ return null;
+ }
+ if (comparator.equals(COMPARATOR_MEMCMP)) {
+ // default comparator
+ return new BytesComparator(new MemcmpRawComparator());
+ } else if (comparator.startsWith(COMPARATOR_JCLASS)) {
+ String compClassName = comparator.substring(COMPARATOR_JCLASS.length()).trim();
+ try {
+ Class compClass = Class.forName(compClassName);
+ // use its default ctor to create an instance
+ return new BytesComparator((RawComparator<Object>) compClass.newInstance());
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Failed to instantiate comparator: " + comparator + "(" + e.toString() + ")");
+ }
+ } else {
+ throw new IllegalArgumentException("Unsupported comparator: " + comparator);
+ }
+ }
+
+ public void write(DataOutput out) throws IOException {
+ TFile.API_VERSION.write(out);
+ Utils.writeVLong(out, recordCount);
+ Utils.writeString(out, strComparator);
+ }
+
+ public long getRecordCount() {
+ return recordCount;
+ }
+
+ public void incRecordCount() {
+ ++recordCount;
+ }
+
+ public boolean isSorted() {
+ return !strComparator.equals("");
+ }
+
+ public String getComparatorString() {
+ return strComparator;
+ }
+
+ public BytesComparator getComparator() {
+ return comparator;
+ }
+
+ public Version getVersion() {
+ return version;
+ }
+ } // END: class MetaTFileMeta
+
+ /**
+ * Data structure representing "TFile.index" meta block.
+ */
+ static class TFileIndex {
+ final static String BLOCK_NAME = "TFile.index";
+ private ByteArray firstKey;
+ private final ArrayList<TFileIndexEntry> index;
+ private final BytesComparator comparator;
+
+ /**
+ * For reading from file.
- *
- * @throws IOException
+ */
+ public TFileIndex(int entryCount, DataInput in, BytesComparator comparator) throws IOException {
+ index = new ArrayList<TFileIndexEntry>(entryCount);
+ int size = Utils.readVInt(in); // size for the first key entry.
+ if (size > 0) {
+ byte[] buffer = new byte[size];
+ in.readFully(buffer);
+ DataInputStream firstKeyInputStream = new DataInputStream(new ByteArrayInputStream(buffer, 0, size));
+
+ int firstKeyLength = Utils.readVInt(firstKeyInputStream);
+ firstKey = new ByteArray(new byte[firstKeyLength]);
+ firstKeyInputStream.readFully(firstKey.buffer());
+
+ for (int i = 0; i < entryCount; i++) {
+ size = Utils.readVInt(in);
+ if (buffer.length < size) {
+ buffer = new byte[size];
+ }
+ in.readFully(buffer, 0, size);
+ TFileIndexEntry idx = new TFileIndexEntry(new DataInputStream(new ByteArrayInputStream(buffer, 0, size)));
+ index.add(idx);
+ }
+ } else {
+ if (entryCount != 0) {
+ throw new RuntimeException("Internal error");
+ }
+ }
+ this.comparator = comparator;
+ }
+
+ /**
+ * @param key
+ * input key.
+ * @return the ID of the first block that contains key >= input key. Or -1 if no such block exists.
+ */
+ public int lowerBound(RawComparable key) {
+ if (comparator == null) {
+ throw new RuntimeException("Cannot search in unsorted TFile");
+ }
+
+ if (firstKey == null) {
+ return -1; // not found
+ }
+
+ int ret = Utils.lowerBound(index, key, comparator);
+ if (ret == index.size()) {
+ return -1;
+ }
+ return ret;
+ }
+
+ public int upperBound(RawComparable key) {
+ if (comparator == null) {
+ throw new RuntimeException("Cannot search in unsorted TFile");
+ }
+
+ if (firstKey == null) {
+ return -1; // not found
+ }
+
+ int ret = Utils.upperBound(index, key, comparator);
+ if (ret == index.size()) {
+ return -1;
+ }
+ return ret;
+ }
+
+ /**
+ * For writing to file.
+ */
+ public TFileIndex(BytesComparator comparator) {
+ index = new ArrayList<TFileIndexEntry>();
+ this.comparator = comparator;
+ }
+
+ public RawComparable getFirstKey() {
+ return firstKey;
+ }
+
+ public void setFirstKey(byte[] key, int offset, int length) {
+ firstKey = new ByteArray(new byte[length]);
+ System.arraycopy(key, offset, firstKey.buffer(), 0, length);
+ }
+
+ public RawComparable getLastKey() {
+ if (index.size() == 0) {
+ return null;
+ }
+ return new ByteArray(index.get(index.size() - 1).buffer());
+ }
+
+ public void addEntry(TFileIndexEntry keyEntry) {
+ index.add(keyEntry);
+ }
+
+ public TFileIndexEntry getEntry(int bid) {
+ return index.get(bid);
+ }
+
+ public void write(DataOutput out) throws IOException {
+ if (firstKey == null) {
+ Utils.writeVInt(out, 0);
+ return;
+ }
+
+ DataOutputBuffer dob = new DataOutputBuffer();
+ Utils.writeVInt(dob, firstKey.size());
+ dob.write(firstKey.buffer());
+ Utils.writeVInt(out, dob.size());
+ out.write(dob.getData(), 0, dob.getLength());
+
+ for (TFileIndexEntry entry : index) {
+ dob.reset();
+ entry.write(dob);
+ Utils.writeVInt(out, dob.getLength());
+ out.write(dob.getData(), 0, dob.getLength());
+ }
+ }
+ }
+
+ /**
+ * TFile Data Index entry. We should try to make the memory footprint of each index entry as small as possible.
+ */
+ static final class TFileIndexEntry implements RawComparable {
+ final byte[] key;
+ // count of <key, value> entries in the block.
+ final long kvEntries;
+
+ public TFileIndexEntry(DataInput in) throws IOException {
+ int len = Utils.readVInt(in);
+ key = new byte[len];
+ in.readFully(key, 0, len);
+ kvEntries = Utils.readVLong(in);
+ }
+
+ // default entry, without any padding
+ public TFileIndexEntry(byte[] newkey, int offset, int len, long entries) {
+ key = new byte[len];
+ System.arraycopy(newkey, offset, key, 0, len);
+ this.kvEntries = entries;
+ }
+
+ @Override
+ public byte[] buffer() {
+ return key;
+ }
+
+ @Override
+ public int offset() {
+ return 0;
+ }
+
+ @Override
+ public int size() {
+ return key.length;
+ }
+
+ long entries() {
+ return kvEntries;
+ }
+
+ public void write(DataOutput out) throws IOException {
+ Utils.writeVInt(out, key.length);
+ out.write(key, 0, key.length);
+ Utils.writeVLong(out, kvEntries);
+ }
+ }
+
+ /**
+ * Dumping the TFile information.
+ *
+ * @param args
+ * A list of TFile paths.
+ */
+ public static void main(String[] args) {
+ System.out.printf("TFile Dumper (TFile %s, BCFile %s)%n", TFile.API_VERSION.toString(), BCFile.API_VERSION.toString());
+ if (args.length == 0) {
+ System.out.println("Usage: java ... org.apache.hadoop.io.file.tfile.TFile tfile-path [tfile-path ...]");
+ System.exit(0);
+ }
+ Configuration conf = new Configuration();
+
+ for (String file : args) {
+ System.out.println("===" + file + "===");
+ try {
+ TFileDumper.dumpInfo(file, System.out, conf);
+ } catch (IOException e) {
+ e.printStackTrace(System.err);
+ }
+ }
+ }
+}
[37/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
index d018228,0000000..f4a8082
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
@@@ -1,136 -1,0 +1,135 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+public class AddFilesWithMissingEntries {
+
+ static final Logger log = Logger.getLogger(AddFilesWithMissingEntries.class);
+
+ public static class Opts extends ClientOpts {
+ @Parameter(names="-update", description="Make changes to the !METADATA table to include missing files")
+ boolean update = false;
+ }
+
+
+ /**
+ * A utility to add files to the !METADATA table that are not listed in the root tablet.
+ * This is a recovery tool for someone who knows what they are doing. It might be better to
+ * save off files, and recover your instance by re-initializing and importing the existing files.
+ *
- * @param args
+ */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(AddFilesWithMissingEntries.class.getName(), args, bwOpts);
+
+ final Key rootTableEnd = new Key(Constants.ROOT_TABLET_EXTENT.getEndRow());
+ final Range range = new Range(rootTableEnd.followingKey(PartialKey.ROW), true, Constants.METADATA_RESERVED_KEYSPACE_START_KEY, false);
+ final Scanner scanner = opts.getConnector().createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ scanner.setRange(range);
+ final Configuration conf = new Configuration();
+ final FileSystem fs = FileSystem.get(conf);
+
+ KeyExtent last = new KeyExtent();
+ String directory = null;
+ Set<String> knownFiles = new HashSet<String>();
+
+ int count = 0;
+ final MultiTableBatchWriter writer = opts.getConnector().createMultiTableBatchWriter(bwOpts.getBatchWriterConfig());
+
+ // collect the list of known files and the directory for each extent
+ for (Entry<Key,Value> entry : scanner) {
+ Key key = entry.getKey();
+ KeyExtent ke = new KeyExtent(key.getRow(), (Text) null);
+ // when the key extent changes
+ if (!ke.equals(last)) {
+ if (directory != null) {
+ // add any files in the directory unknown to the key extent
+ count += addUnknownFiles(fs, directory, knownFiles, last, writer, opts.update);
+ }
+ directory = null;
+ knownFiles.clear();
+ last = ke;
+ }
+ if (Constants.METADATA_DIRECTORY_COLUMN.hasColumns(key)) {
+ directory = entry.getValue().toString();
+ log.debug("Found directory " + directory + " for row " + key.getRow().toString());
+ } else if (key.compareColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY) == 0) {
+ String filename = key.getColumnQualifier().toString();
+ knownFiles.add(filename);
+ log.debug("METADATA file found: " + filename);
+ }
+ }
+ if (directory != null) {
+ // catch the last key extent
+ count += addUnknownFiles(fs, directory, knownFiles, last, writer, opts.update);
+ }
+ log.info("There were " + count + " files that are unknown to the metadata table");
+ writer.close();
+ }
+
+ private static int addUnknownFiles(FileSystem fs, String directory, Set<String> knownFiles, KeyExtent ke, MultiTableBatchWriter writer, boolean update) throws Exception {
+ int count = 0;
+ final String tableId = ke.getTableId().toString();
+ final Text row = ke.getMetadataEntry();
+ log.info(row.toString());
+ final Path path = new Path(ServerConstants.getTablesDir() + "/" + tableId + directory);
+ for (FileStatus file : fs.listStatus(path)) {
+ if (file.getPath().getName().endsWith("_tmp") || file.getPath().getName().endsWith("_tmp.rf"))
+ continue;
+ final String filename = directory + "/" + file.getPath().getName();
+ if (!knownFiles.contains(filename)) {
+ count++;
+ final Mutation m = new Mutation(row);
+ String size = Long.toString(file.getLen());
+ String entries = "1"; // lie
+ String value = size + "," + entries;
+ m.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(filename), new Value(value.getBytes(Constants.UTF8)));
+ if (update) {
+ writer.getBatchWriter(Constants.METADATA_TABLE_NAME).addMutation(m);
+ }
+ }
+ }
+ return count;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
index 95f6a32,0000000..3342993
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
@@@ -1,123 -1,0 +1,120 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+
+import com.beust.jcommander.Parameter;
+
+public class DumpZookeeper {
+
+ static IZooReaderWriter zk = null;
+
+ private static final Logger log = Logger.getLogger(DumpZookeeper.class);
+
+ private static class Encoded {
+ public String encoding;
+ public String value;
+
+ Encoded(String e, String v) {
+ encoding = e;
+ value = v;
+ }
+ }
+
+ static class Opts extends Help {
+ @Parameter(names="--root", description="the root of the znode tree to dump")
+ String root = "/";
+ }
+
- /**
- * @param args
- */
+ public static void main(String[] args) {
+ Opts opts = new Opts();
+ opts.parseArgs(DumpZookeeper.class.getName(), args);
+
+ Logger.getRootLogger().setLevel(Level.WARN);
+ PrintStream out = System.out;
+ try {
+ zk = ZooReaderWriter.getInstance();
+
+ write(out, 0, "<dump root='%s'>", opts.root);
+ for (String child : zk.getChildren(opts.root, null))
+ if (!child.equals("zookeeper"))
+ dump(out, opts.root, child, 1);
+ write(out, 0, "</dump>");
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ }
+
+ private static void dump(PrintStream out, String root, String child, int indent) throws KeeperException, InterruptedException, UnsupportedEncodingException {
+ String path = root + "/" + child;
+ if (root.endsWith("/"))
+ path = root + child;
+ Stat stat = zk.getStatus(path);
+ if (stat == null)
+ return;
+ String type = "node";
+ if (stat.getEphemeralOwner() != 0) {
+ type = "ephemeral";
+ }
+ if (stat.getNumChildren() == 0) {
+ if (stat.getDataLength() == 0) {
+ write(out, indent, "<%s name='%s'/>", type, child);
+ } else {
+ Encoded value = value(path);
+ write(out, indent, "<%s name='%s' encoding='%s' value='%s'/>", type, child, value.encoding, value.value);
+ }
+ } else {
+ if (stat.getDataLength() == 0) {
+ write(out, indent, "<%s name='%s'>", type, child);
+ } else {
+ Encoded value = value(path);
+ write(out, indent, "<%s name='%s' encoding='%s' value='%s'>", type, child, value.encoding, value.value);
+ }
+ for (String c : zk.getChildren(path, null)) {
+ dump(out, path, c, indent + 1);
+ }
+ write(out, indent, "</node>");
+ }
+ }
+
+ private static Encoded value(String path) throws KeeperException, InterruptedException, UnsupportedEncodingException {
+ byte[] data = zk.getData(path, null);
+ for (int i = 0; i < data.length; i++) {
+ // does this look like simple ascii?
+ if (data[i] < ' ' || data[i] > '~')
+ return new Encoded("base64", new String(Base64.encodeBase64(data), "utf8"));
+ }
+ return new Encoded("utf8", new String(data, "utf8"));
+ }
+
+ private static void write(PrintStream out, int indent, String fmt, Object... args) {
+ for (int i = 0; i < indent; i++)
+ out.print(" ");
+ out.println(String.format(fmt, args));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
index 60b50da,0000000..42ebbe2
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
@@@ -1,74 -1,0 +1,71 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.util.Iterator;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.server.master.LiveTServerSet;
+import org.apache.accumulo.server.master.LiveTServerSet.Listener;
+import org.apache.accumulo.server.master.state.MetaDataTableScanner;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletLocationState;
+import org.apache.accumulo.server.master.state.TabletState;
+import org.apache.accumulo.server.master.state.tables.TableManager;
+import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.commons.collections.iterators.IteratorChain;
+import org.apache.log4j.Logger;
+
+public class FindOfflineTablets {
+ private static final Logger log = Logger.getLogger(FindOfflineTablets.class);
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ ClientOpts opts = new ClientOpts();
+ opts.parseArgs(FindOfflineTablets.class.getName(), args);
+ final AtomicBoolean scanning = new AtomicBoolean(false);
+ Instance instance = opts.getInstance();
+ MetaDataTableScanner rootScanner = new MetaDataTableScanner(instance, SecurityConstants.getSystemCredentials(), Constants.METADATA_ROOT_TABLET_KEYSPACE);
+ MetaDataTableScanner metaScanner = new MetaDataTableScanner(instance, SecurityConstants.getSystemCredentials(), Constants.NON_ROOT_METADATA_KEYSPACE);
+ @SuppressWarnings("unchecked")
+ Iterator<TabletLocationState> scanner = (Iterator<TabletLocationState>)new IteratorChain(rootScanner, metaScanner);
+ LiveTServerSet tservers = new LiveTServerSet(instance, DefaultConfiguration.getDefaultConfiguration(), new Listener() {
+ @Override
+ public void update(LiveTServerSet current, Set<TServerInstance> deleted, Set<TServerInstance> added) {
+ if (!deleted.isEmpty() && scanning.get())
+ log.warn("Tablet servers deleted while scanning: " + deleted);
+ if (!added.isEmpty() && scanning.get())
+ log.warn("Tablet servers added while scanning: " + added);
+ }
+ });
+ tservers.startListeningForTabletServerChanges();
+ scanning.set(true);
+ while (scanner.hasNext()) {
+ TabletLocationState locationState = scanner.next();
+ TabletState state = locationState.getState(tservers.getCurrentServers());
+ if (state != null && state != TabletState.HOSTED && TableManager.getInstance().getTableState(locationState.extent.getTableId().toString()) != TableState.OFFLINE)
+ if (!locationState.extent.equals(Constants.ROOT_TABLET_EXTENT))
+ System.out.println(locationState + " is " + state + " #walogs:" + locationState.walogs.size());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
index e16bd06,0000000..cf1a065
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/LoginProperties.java
@@@ -1,62 -1,0 +1,59 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.TokenProperty;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.security.handler.Authenticator;
+import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+
+/**
+ *
+ */
+public class LoginProperties {
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ AccumuloConfiguration config = ServerConfiguration.getSystemConfiguration(HdfsZooInstance.getInstance());
+ Authenticator authenticator = AccumuloClassLoader.getClassLoader().loadClass(config.get(Property.INSTANCE_SECURITY_AUTHENTICATOR))
+ .asSubclass(Authenticator.class).newInstance();
+
+ List<Set<TokenProperty>> tokenProps = new ArrayList<Set<TokenProperty>>();
+
+ for (Class<? extends AuthenticationToken> tokenType : authenticator.getSupportedTokenTypes()) {
+ tokenProps.add(tokenType.newInstance().getProperties());
+ }
+
+ System.out.println("Supported token types for " + authenticator.getClass().getName() + " are : ");
+ for (Class<? extends AuthenticationToken> tokenType : authenticator.getSupportedTokenTypes()) {
+ System.out.println("\t" + tokenType.getName() + ", which accepts the following properties : ");
+
+ for (TokenProperty tokenProperty : tokenType.newInstance().getProperties()) {
+ System.out.println("\t\t" + tokenProperty);
+ }
+
+ System.out.println();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
index d6e0a3c,0000000..477718d
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
@@@ -1,1262 -1,0 +1,1257 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.BatchWriterImpl;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
+import org.apache.accumulo.core.client.impl.Writer;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.accumulo.core.util.FastFormat;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.StringUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.accumulo.server.trace.TraceFileSystem;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * provides a reference to the metadata table for updates by tablet servers
+ */
+public class MetadataTable extends org.apache.accumulo.core.util.MetadataTable {
-
++
+ private static final Text EMPTY_TEXT = new Text();
+ private static Map<TCredentials,Writer> metadata_tables = new HashMap<TCredentials,Writer>();
+ private static final Logger log = Logger.getLogger(MetadataTable.class);
-
++
+ private static final int SAVE_ROOT_TABLET_RETRIES = 3;
-
++
+ private MetadataTable() {
-
++
+ }
+
+ public synchronized static Writer getMetadataTable(TCredentials credentials) {
+ Writer metadataTable = metadata_tables.get(credentials);
+ if (metadataTable == null) {
+ metadataTable = new Writer(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID);
+ metadata_tables.put(credentials, metadataTable);
+ }
+ return metadataTable;
+ }
-
++
+ public static void putLockID(ZooLock zooLock, Mutation m) {
+ Constants.METADATA_LOCK_COLUMN.put(m, new Value(zooLock.getLockID().serialize(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + "/").getBytes(Constants.UTF8)));
+ }
+
+ public static void update(TCredentials credentials, Mutation m) {
+ update(credentials, null, m);
+ }
+
+ public static void update(TCredentials credentials, ZooLock zooLock, Mutation m) {
+ Writer t;
+ t = getMetadataTable(credentials);
+ if (zooLock != null)
+ putLockID(zooLock, m);
+ while (true) {
+ try {
+ t.update(m);
+ return;
+ } catch (AccumuloException e) {
+ log.error(e, e);
+ } catch (AccumuloSecurityException e) {
+ log.error(e, e);
+ } catch (ConstraintViolationException e) {
+ log.error(e, e);
+ } catch (TableNotFoundException e) {
+ log.error(e, e);
+ }
+ UtilWaitThread.sleep(1000);
+ }
-
++
+ }
-
++
+ /**
+ * new data file update function adds one data file to a tablet's list
- *
- * path should be relative to the table directory
- *
- * @param time
- * @param filesInUseByScans
- * @param zooLock
- * @param flushId
- *
++ *
++ * @param path
++ * should be relative to the table directory
+ */
+ public static void updateTabletDataFile(KeyExtent extent, String path, String mergeFile, DataFileValue dfv, String time, TCredentials credentials,
+ Set<String> filesInUseByScans, String address, ZooLock zooLock, Set<String> unusedWalLogs, TServerInstance lastLocation, long flushId) {
+ if (extent.equals(Constants.ROOT_TABLET_EXTENT)) {
+ if (unusedWalLogs != null) {
+ IZooReaderWriter zk = ZooReaderWriter.getInstance();
+ // unusedWalLogs will contain the location/name of each log in a log set
+ // the log set is stored under one of the log names, but not both
+ // find the entry under one of the names and delete it.
+ String root = getZookeeperLogLocation();
+ boolean foundEntry = false;
+ for (String entry : unusedWalLogs) {
+ String[] parts = entry.split("/");
+ String zpath = root + "/" + parts[1];
+ while (true) {
+ try {
+ if (zk.exists(zpath)) {
+ zk.recursiveDelete(zpath, NodeMissingPolicy.SKIP);
+ foundEntry = true;
+ }
+ break;
+ } catch (KeeperException e) {
+ log.error(e, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ }
+ UtilWaitThread.sleep(1000);
+ }
+ }
+ if (unusedWalLogs.size() > 0 && !foundEntry)
+ log.warn("WALog entry for root tablet did not exist " + unusedWalLogs);
+ }
+ return;
+ }
-
++
+ Mutation m = new Mutation(extent.getMetadataEntry());
-
++
+ if (dfv.getNumEntries() > 0) {
+ m.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(path), new Value(dfv.encode()));
+ Constants.METADATA_TIME_COLUMN.put(m, new Value(time.getBytes(Constants.UTF8)));
+ // stuff in this location
+ TServerInstance self = getTServerInstance(address, zooLock);
+ self.putLastLocation(m);
+ // erase the old location
+ if (lastLocation != null && !lastLocation.equals(self))
+ lastLocation.clearLastLocation(m);
+ }
+ if (unusedWalLogs != null) {
+ for (String entry : unusedWalLogs) {
+ m.putDelete(Constants.METADATA_LOG_COLUMN_FAMILY, new Text(entry));
+ }
+ }
-
++
+ for (String scanFile : filesInUseByScans)
+ m.put(Constants.METADATA_SCANFILE_COLUMN_FAMILY, new Text(scanFile), new Value(new byte[0]));
+
+ if (mergeFile != null)
+ m.putDelete(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(mergeFile));
+
+ Constants.METADATA_FLUSH_COLUMN.put(m, new Value(Long.toString(flushId).getBytes(Constants.UTF8)));
+
+ update(credentials, zooLock, m);
-
++
+ }
-
++
+ private static TServerInstance getTServerInstance(String address, ZooLock zooLock) {
+ while (true) {
+ try {
+ return new TServerInstance(address, zooLock.getSessionId());
+ } catch (KeeperException e) {
+ log.error(e, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ }
+ UtilWaitThread.sleep(1000);
+ }
+ }
+
+ public static void updateTabletFlushID(KeyExtent extent, long flushID, TCredentials credentials, ZooLock zooLock) {
+ if (!extent.isRootTablet()) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
+ Constants.METADATA_FLUSH_COLUMN.put(m, new Value(Long.toString(flushID).getBytes(Constants.UTF8)));
+ update(credentials, zooLock, m);
+ }
+ }
+
+ public static void updateTabletCompactID(KeyExtent extent, long compactID, TCredentials credentials, ZooLock zooLock) {
+ if (!extent.isRootTablet()) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
+ Constants.METADATA_COMPACT_COLUMN.put(m, new Value(Long.toString(compactID).getBytes(Constants.UTF8)));
+ update(credentials, zooLock, m);
+ }
+ }
+
+ public static void updateTabletDataFile(long tid, KeyExtent extent, Map<String,DataFileValue> estSizes, String time, TCredentials credentials, ZooLock zooLock) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
+ byte[] tidBytes = Long.toString(tid).getBytes(Constants.UTF8);
+
+ for (Entry<String,DataFileValue> entry : estSizes.entrySet()) {
+ Text file = new Text(entry.getKey());
+ m.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, file, new Value(entry.getValue().encode()));
+ m.put(Constants.METADATA_BULKFILE_COLUMN_FAMILY, file, new Value(tidBytes));
+ }
+ Constants.METADATA_TIME_COLUMN.put(m, new Value(time.getBytes(Constants.UTF8)));
+ update(credentials, zooLock, m);
+ }
+
+ public static void addTablet(KeyExtent extent, String path, TCredentials credentials, char timeType, ZooLock lock) {
+ Mutation m = extent.getPrevRowUpdateMutation();
+
+ Constants.METADATA_DIRECTORY_COLUMN.put(m, new Value(path.getBytes(Constants.UTF8)));
+ Constants.METADATA_TIME_COLUMN.put(m, new Value((timeType + "0").getBytes(Constants.UTF8)));
+
+ update(credentials, lock, m);
+ }
+
+ public static void updateTabletPrevEndRow(KeyExtent extent, TCredentials credentials) {
+ Mutation m = extent.getPrevRowUpdateMutation(); //
+ update(credentials, m);
+ }
-
++
+ /**
+ * convenience method for reading entries from the metadata table
+ */
+ public static SortedMap<KeyExtent,Text> getMetadataDirectoryEntries(SortedMap<Key,Value> entries) {
+ Key key;
+ Value val;
+ Text datafile = null;
+ Value prevRow = null;
+ KeyExtent ke;
-
++
+ SortedMap<KeyExtent,Text> results = new TreeMap<KeyExtent,Text>();
-
++
+ Text lastRowFromKey = new Text();
-
++
+ // text obj below is meant to be reused in loop for efficiency
+ Text colf = new Text();
+ Text colq = new Text();
-
++
+ for (Entry<Key,Value> entry : entries.entrySet()) {
+ key = entry.getKey();
+ val = entry.getValue();
-
++
+ if (key.compareRow(lastRowFromKey) != 0) {
+ prevRow = null;
+ datafile = null;
+ key.getRow(lastRowFromKey);
+ }
-
++
+ colf = key.getColumnFamily(colf);
+ colq = key.getColumnQualifier(colq);
-
++
+ // interpret the row id as a key extent
+ if (Constants.METADATA_DIRECTORY_COLUMN.equals(colf, colq))
+ datafile = new Text(val.toString());
-
++
+ else if (Constants.METADATA_PREV_ROW_COLUMN.equals(colf, colq))
+ prevRow = new Value(val);
-
++
+ if (datafile != null && prevRow != null) {
+ ke = new KeyExtent(key.getRow(), prevRow);
+ results.put(ke, datafile);
-
++
+ datafile = null;
+ prevRow = null;
+ }
+ }
+ return results;
+ }
+
+ public static boolean recordRootTabletLocation(String address) {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ for (int i = 0; i < SAVE_ROOT_TABLET_RETRIES; i++) {
+ try {
+ log.info("trying to write root tablet location to ZooKeeper as " + address);
+ String zRootLocPath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZROOT_TABLET_LOCATION;
+ zoo.putPersistentData(zRootLocPath, address.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
+ return true;
+ } catch (Exception e) {
+ log.error("Master: unable to save root tablet location in zookeeper. exception: " + e, e);
+ }
+ }
+ log.error("Giving up after " + SAVE_ROOT_TABLET_RETRIES + " retries");
+ return false;
+ }
+
+ public static SortedMap<String,DataFileValue> getDataFileSizes(KeyExtent extent, TCredentials credentials) {
+ TreeMap<String,DataFileValue> sizes = new TreeMap<String,DataFileValue>();
-
++
+ Scanner mdScanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ mdScanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
+ Text row = extent.getMetadataEntry();
-
++
+ Key endKey = new Key(row, Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(""));
+ endKey = endKey.followingKey(PartialKey.ROW_COLFAM);
-
++
+ mdScanner.setRange(new Range(new Key(row), endKey));
+ for (Entry<Key,Value> entry : mdScanner) {
-
++
+ if (!entry.getKey().getRow().equals(row))
+ break;
+ DataFileValue dfv = new DataFileValue(entry.getValue().get());
+ sizes.put(entry.getKey().getColumnQualifier().toString(), dfv);
+ }
-
++
+ return sizes;
+ }
-
++
+ public static void addNewTablet(KeyExtent extent, String path, TServerInstance location, Map<String,DataFileValue> datafileSizes,
+ Map<String,Long> bulkLoadedFiles, TCredentials credentials, String time, long lastFlushID, long lastCompactID, ZooLock zooLock) {
+ Mutation m = extent.getPrevRowUpdateMutation();
+
+ Constants.METADATA_DIRECTORY_COLUMN.put(m, new Value(path.getBytes(Constants.UTF8)));
+ Constants.METADATA_TIME_COLUMN.put(m, new Value(time.getBytes(Constants.UTF8)));
+ if (lastFlushID > 0)
+ Constants.METADATA_FLUSH_COLUMN.put(m, new Value(Long.toString(lastFlushID).getBytes(Constants.UTF8)));
+ if (lastCompactID > 0)
+ Constants.METADATA_COMPACT_COLUMN.put(m, new Value(Long.toString(lastCompactID).getBytes(Constants.UTF8)));
+
+ if (location != null) {
+ m.put(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY, location.asColumnQualifier(), location.asMutationValue());
+ m.putDelete(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY, location.asColumnQualifier());
+ }
-
++
+ for (Entry<String,DataFileValue> entry : datafileSizes.entrySet()) {
+ m.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(entry.getKey()), new Value(entry.getValue().encode()));
+ }
-
++
+ for (Entry<String,Long> entry : bulkLoadedFiles.entrySet()) {
+ byte[] tidBytes = Long.toString(entry.getValue()).getBytes(Constants.UTF8);
+ m.put(Constants.METADATA_BULKFILE_COLUMN_FAMILY, new Text(entry.getKey()), new Value(tidBytes));
+ }
+
+ update(credentials, zooLock, m);
+ }
+
+ public static void rollBackSplit(Text metadataEntry, Text oldPrevEndRow, TCredentials credentials, ZooLock zooLock) {
+ KeyExtent ke = new KeyExtent(metadataEntry, oldPrevEndRow);
+ Mutation m = ke.getPrevRowUpdateMutation();
+ Constants.METADATA_SPLIT_RATIO_COLUMN.putDelete(m);
+ Constants.METADATA_OLD_PREV_ROW_COLUMN.putDelete(m);
+ update(credentials, zooLock, m);
+ }
+
+ public static void splitTablet(KeyExtent extent, Text oldPrevEndRow, double splitRatio, TCredentials credentials, ZooLock zooLock) {
+ Mutation m = extent.getPrevRowUpdateMutation(); //
+
+ Constants.METADATA_SPLIT_RATIO_COLUMN.put(m, new Value(Double.toString(splitRatio).getBytes(Constants.UTF8)));
+
+ Constants.METADATA_OLD_PREV_ROW_COLUMN.put(m, KeyExtent.encodePrevEndRow(oldPrevEndRow));
+ Constants.METADATA_CHOPPED_COLUMN.putDelete(m);
+ update(credentials, zooLock, m);
+ }
+
+ public static void finishSplit(Text metadataEntry, Map<String,DataFileValue> datafileSizes, List<String> highDatafilesToRemove, TCredentials credentials,
+ ZooLock zooLock) {
+ Mutation m = new Mutation(metadataEntry);
+ Constants.METADATA_SPLIT_RATIO_COLUMN.putDelete(m);
+ Constants.METADATA_OLD_PREV_ROW_COLUMN.putDelete(m);
+ Constants.METADATA_CHOPPED_COLUMN.putDelete(m);
+
+ for (Entry<String,DataFileValue> entry : datafileSizes.entrySet()) {
+ m.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(entry.getKey()), new Value(entry.getValue().encode()));
+ }
-
++
+ for (String pathToRemove : highDatafilesToRemove) {
+ m.putDelete(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(pathToRemove));
+ }
-
++
+ update(credentials, zooLock, m);
+ }
+
+ public static void finishSplit(KeyExtent extent, Map<String,DataFileValue> datafileSizes, List<String> highDatafilesToRemove, TCredentials credentials,
+ ZooLock zooLock) {
+ finishSplit(extent.getMetadataEntry(), datafileSizes, highDatafilesToRemove, credentials, zooLock);
+ }
-
++
+ public static void replaceDatafiles(KeyExtent extent, Set<String> datafilesToDelete, Set<String> scanFiles, String path, Long compactionId,
+ DataFileValue size, TCredentials credentials, String address, TServerInstance lastLocation, ZooLock zooLock) {
+ replaceDatafiles(extent, datafilesToDelete, scanFiles, path, compactionId, size, credentials, address, lastLocation, zooLock, true);
+ }
-
++
+ public static void replaceDatafiles(KeyExtent extent, Set<String> datafilesToDelete, Set<String> scanFiles, String path, Long compactionId,
+ DataFileValue size, TCredentials credentials, String address, TServerInstance lastLocation, ZooLock zooLock, boolean insertDeleteFlags) {
+
+ if (insertDeleteFlags) {
+ // add delete flags for those paths before the data file reference is removed
+ addDeleteEntries(extent, datafilesToDelete, credentials);
+ }
-
++
+ // replace data file references to old mapfiles with the new mapfiles
+ Mutation m = new Mutation(extent.getMetadataEntry());
-
++
+ for (String pathToRemove : datafilesToDelete)
+ m.putDelete(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(pathToRemove));
-
++
+ for (String scanFile : scanFiles)
+ m.put(Constants.METADATA_SCANFILE_COLUMN_FAMILY, new Text(scanFile), new Value(new byte[0]));
+
+ if (size.getNumEntries() > 0)
+ m.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(path), new Value(size.encode()));
-
++
+ if (compactionId != null)
+ Constants.METADATA_COMPACT_COLUMN.put(m, new Value(Long.toString(compactionId).getBytes(Constants.UTF8)));
+
+ TServerInstance self = getTServerInstance(address, zooLock);
+ self.putLastLocation(m);
+
+ // remove the old location
+ if (lastLocation != null && !lastLocation.equals(self))
+ lastLocation.clearLastLocation(m);
+
+ update(credentials, zooLock, m);
+ }
+
+ public static void addDeleteEntries(KeyExtent extent, Set<String> datafilesToDelete, TCredentials credentials) {
+
+ String tableId = extent.getTableId().toString();
+
+ // TODO could use batch writer,would need to handle failure and retry like update does - ACCUMULO-1294
+ for (String pathToRemove : datafilesToDelete)
+ update(credentials, createDeleteMutation(tableId, pathToRemove));
+ }
-
++
+ public static void addDeleteEntry(String tableId, String path) {
+ update(SecurityConstants.getSystemCredentials(), createDeleteMutation(tableId, path));
+ }
-
++
+ public static Mutation createDeleteMutation(String tableId, String pathToRemove) {
+ Mutation delFlag;
+ String prefix = Constants.METADATA_DELETE_FLAG_PREFIX;
+ if (tableId.equals(Constants.METADATA_TABLE_ID))
+ prefix = Constants.METADATA_DELETE_FLAG_FOR_METADATA_PREFIX;
+
+ if (pathToRemove.startsWith("../"))
+ delFlag = new Mutation(new Text(prefix + pathToRemove.substring(2)));
+ else
+ delFlag = new Mutation(new Text(prefix + "/" + tableId + pathToRemove));
+
+ delFlag.put(EMPTY_TEXT, EMPTY_TEXT, new Value(new byte[] {}));
+ return delFlag;
+ }
+
+ public static void removeScanFiles(KeyExtent extent, Set<String> scanFiles, TCredentials credentials, ZooLock zooLock) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
-
++
+ for (String pathToRemove : scanFiles)
+ m.putDelete(Constants.METADATA_SCANFILE_COLUMN_FAMILY, new Text(pathToRemove));
-
++
+ update(credentials, zooLock, m);
+ }
+
+ private static KeyExtent fixSplit(Text table, Text metadataEntry, Text metadataPrevEndRow, Value oper, double splitRatio, TServerInstance tserver,
+ TCredentials credentials, String time, long initFlushID, long initCompactID, ZooLock lock) throws AccumuloException {
+ if (metadataPrevEndRow == null)
+ // something is wrong, this should not happen... if a tablet is split, it will always have a
+ // prev end row....
+ throw new AccumuloException("Split tablet does not have prev end row, something is amiss, extent = " + metadataEntry);
+
+ // check to see if prev tablet exist in metadata tablet
+ Key prevRowKey = new Key(new Text(KeyExtent.getMetadataEntry(table, metadataPrevEndRow)));
+
+ ScannerImpl scanner2 = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ scanner2.setRange(new Range(prevRowKey, prevRowKey.followingKey(PartialKey.ROW)));
-
++
+ if (!scanner2.iterator().hasNext()) {
+ log.info("Rolling back incomplete split " + metadataEntry + " " + metadataPrevEndRow);
+ rollBackSplit(metadataEntry, KeyExtent.decodePrevEndRow(oper), credentials, lock);
+ return new KeyExtent(metadataEntry, KeyExtent.decodePrevEndRow(oper));
+ } else {
+ log.info("Finishing incomplete split " + metadataEntry + " " + metadataPrevEndRow);
+
+ List<String> highDatafilesToRemove = new ArrayList<String>();
+
+ Scanner scanner3 = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ Key rowKey = new Key(metadataEntry);
+
+ SortedMap<String,DataFileValue> origDatafileSizes = new TreeMap<String,DataFileValue>();
+ SortedMap<String,DataFileValue> highDatafileSizes = new TreeMap<String,DataFileValue>();
+ SortedMap<String,DataFileValue> lowDatafileSizes = new TreeMap<String,DataFileValue>();
+ scanner3.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
+ scanner3.setRange(new Range(rowKey, rowKey.followingKey(PartialKey.ROW)));
+
+ for (Entry<Key,Value> entry : scanner3) {
+ if (entry.getKey().compareColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY) == 0) {
+ origDatafileSizes.put(entry.getKey().getColumnQualifier().toString(), new DataFileValue(entry.getValue().get()));
+ }
+ }
+
+ splitDatafiles(table, metadataPrevEndRow, splitRatio, new HashMap<String,FileUtil.FileInfo>(), origDatafileSizes, lowDatafileSizes, highDatafileSizes,
+ highDatafilesToRemove);
+
+ MetadataTable.finishSplit(metadataEntry, highDatafileSizes, highDatafilesToRemove, credentials, lock);
+
+ return new KeyExtent(metadataEntry, KeyExtent.encodePrevEndRow(metadataPrevEndRow));
+ }
+
+
+ }
-
++
+ public static void splitDatafiles(Text table, Text midRow, double splitRatio, Map<String,FileUtil.FileInfo> firstAndLastRows,
+ SortedMap<String,DataFileValue> datafiles, SortedMap<String,DataFileValue> lowDatafileSizes, SortedMap<String,DataFileValue> highDatafileSizes,
+ List<String> highDatafilesToRemove) {
-
++
+ for (Entry<String,DataFileValue> entry : datafiles.entrySet()) {
-
++
+ Text firstRow = null;
+ Text lastRow = null;
-
++
+ boolean rowsKnown = false;
-
++
+ FileUtil.FileInfo mfi = firstAndLastRows.get(entry.getKey());
-
++
+ if (mfi != null) {
+ firstRow = mfi.getFirstRow();
+ lastRow = mfi.getLastRow();
+ rowsKnown = true;
+ }
-
++
+ if (rowsKnown && firstRow.compareTo(midRow) > 0) {
+ // only in high
+ long highSize = entry.getValue().getSize();
+ long highEntries = entry.getValue().getNumEntries();
+ highDatafileSizes.put(entry.getKey(), new DataFileValue(highSize, highEntries, entry.getValue().getTime()));
+ } else if (rowsKnown && lastRow.compareTo(midRow) <= 0) {
+ // only in low
+ long lowSize = entry.getValue().getSize();
+ long lowEntries = entry.getValue().getNumEntries();
+ lowDatafileSizes.put(entry.getKey(), new DataFileValue(lowSize, lowEntries, entry.getValue().getTime()));
-
++
+ highDatafilesToRemove.add(entry.getKey());
+ } else {
+ long lowSize = (long) Math.floor((entry.getValue().getSize() * splitRatio));
+ long lowEntries = (long) Math.floor((entry.getValue().getNumEntries() * splitRatio));
+ lowDatafileSizes.put(entry.getKey(), new DataFileValue(lowSize, lowEntries, entry.getValue().getTime()));
-
++
+ long highSize = (long) Math.ceil((entry.getValue().getSize() * (1.0 - splitRatio)));
+ long highEntries = (long) Math.ceil((entry.getValue().getNumEntries() * (1.0 - splitRatio)));
+ highDatafileSizes.put(entry.getKey(), new DataFileValue(highSize, highEntries, entry.getValue().getTime()));
+ }
+ }
+ }
+
+ public static KeyExtent fixSplit(Text metadataEntry, SortedMap<ColumnFQ,Value> columns, TServerInstance tserver, TCredentials credentials, ZooLock lock)
+ throws AccumuloException {
+ log.info("Incomplete split " + metadataEntry + " attempting to fix");
+
+ Value oper = columns.get(Constants.METADATA_OLD_PREV_ROW_COLUMN);
-
++
+ if (columns.get(Constants.METADATA_SPLIT_RATIO_COLUMN) == null) {
+ throw new IllegalArgumentException("Metadata entry does not have split ratio (" + metadataEntry + ")");
+ }
+
+ double splitRatio = Double.parseDouble(new String(columns.get(Constants.METADATA_SPLIT_RATIO_COLUMN).get(), Constants.UTF8));
+
+ Value prevEndRowIBW = columns.get(Constants.METADATA_PREV_ROW_COLUMN);
-
++
+ if (prevEndRowIBW == null) {
+ throw new IllegalArgumentException("Metadata entry does not have prev row (" + metadataEntry + ")");
+ }
-
++
+ Value time = columns.get(Constants.METADATA_TIME_COLUMN);
-
++
+ if (time == null) {
+ throw new IllegalArgumentException("Metadata entry does not have time (" + metadataEntry + ")");
+ }
-
++
+ Value flushID = columns.get(Constants.METADATA_FLUSH_COLUMN);
+ long initFlushID = -1;
+ if (flushID != null)
+ initFlushID = Long.parseLong(flushID.toString());
-
++
+ Value compactID = columns.get(Constants.METADATA_COMPACT_COLUMN);
+ long initCompactID = -1;
+ if (compactID != null)
+ initCompactID = Long.parseLong(compactID.toString());
-
++
+ Text metadataPrevEndRow = KeyExtent.decodePrevEndRow(prevEndRowIBW);
-
++
+ Text table = (new KeyExtent(metadataEntry, (Text) null)).getTableId();
-
++
+ return fixSplit(table, metadataEntry, metadataPrevEndRow, oper, splitRatio, tserver, credentials, time.toString(), initFlushID, initCompactID, lock);
+ }
+
+ public static void deleteTable(String tableId, boolean insertDeletes, TCredentials credentials, ZooLock lock) throws AccumuloException {
+ Scanner ms = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ Text tableIdText = new Text(tableId);
+ BatchWriter bw = new BatchWriterImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, new BatchWriterConfig().setMaxMemory(1000000)
+ .setMaxLatency(120000l, TimeUnit.MILLISECONDS).setMaxWriteThreads(2));
+
+ // scan metadata for our table and delete everything we find
+ Mutation m = null;
+ ms.setRange(new KeyExtent(tableIdText, null, null).toMetadataRange());
-
++
+ // insert deletes before deleting data from !METADATA... this makes the code fault tolerant
+ if (insertDeletes) {
-
++
+ ms.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
+ Constants.METADATA_DIRECTORY_COLUMN.fetch(ms);
+
+ for (Entry<Key,Value> cell : ms) {
+ Key key = cell.getKey();
-
++
+ if (key.getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
+ String relPath = key.getColumnQualifier().toString();
+ // only insert deletes for files owned by this table
+ if (!relPath.startsWith("../"))
+ bw.addMutation(createDeleteMutation(tableId, relPath));
+ }
-
++
+ if (Constants.METADATA_DIRECTORY_COLUMN.hasColumns(key)) {
+ bw.addMutation(createDeleteMutation(tableId, cell.getValue().toString()));
+ }
+ }
-
++
+ bw.flush();
-
++
+ ms.clearColumns();
+ }
-
++
+ for (Entry<Key,Value> cell : ms) {
+ Key key = cell.getKey();
-
++
+ if (m == null) {
+ m = new Mutation(key.getRow());
+ if (lock != null)
+ putLockID(lock, m);
+ }
-
++
+ if (key.getRow().compareTo(m.getRow(), 0, m.getRow().length) != 0) {
+ bw.addMutation(m);
+ m = new Mutation(key.getRow());
+ if (lock != null)
+ putLockID(lock, m);
+ }
+ m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
+ }
-
++
+ if (m != null)
+ bw.addMutation(m);
-
++
+ bw.close();
+ }
-
++
+ public static class LogEntry {
+ public KeyExtent extent;
+ public long timestamp;
+ public String server;
+ public String filename;
+ public int tabletId;
+ public Collection<String> logSet;
+
+ @Override
+ public String toString() {
+ return extent.toString() + " " + filename + " (" + tabletId + ")";
+ }
-
++
+ public String getName() {
+ return server + "/" + filename;
+ }
-
++
+ public byte[] toBytes() throws IOException {
+ DataOutputBuffer out = new DataOutputBuffer();
+ extent.write(out);
+ out.writeLong(timestamp);
+ out.writeUTF(server);
+ out.writeUTF(filename);
+ out.write(tabletId);
+ out.write(logSet.size());
+ for (String s : logSet) {
+ out.writeUTF(s);
+ }
+ return Arrays.copyOf(out.getData(), out.getLength());
+ }
-
++
+ public void fromBytes(byte bytes[]) throws IOException {
+ DataInputBuffer inp = new DataInputBuffer();
+ inp.reset(bytes, bytes.length);
+ extent = new KeyExtent();
+ extent.readFields(inp);
+ timestamp = inp.readLong();
+ server = inp.readUTF();
+ filename = inp.readUTF();
+ tabletId = inp.read();
+ int count = inp.read();
+ ArrayList<String> logSet = new ArrayList<String>(count);
+ for (int i = 0; i < count; i++)
+ logSet.add(inp.readUTF());
+ this.logSet = logSet;
+ }
-
++
+ }
-
++
+ private static String getZookeeperLogLocation() {
+ return ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZROOT_TABLET_WALOGS;
+ }
+
+ public static void addLogEntry(TCredentials credentials, LogEntry entry, ZooLock zooLock) {
+ if (entry.extent.isRootTablet()) {
+ String root = getZookeeperLogLocation();
+ while (true) {
+ try {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ if (zoo.isLockHeld(zooLock.getLockID()))
+ zoo.putPersistentData(root + "/" + entry.filename, entry.toBytes(), NodeExistsPolicy.OVERWRITE);
+ break;
+ } catch (KeeperException e) {
+ log.error(e, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ } catch (IOException e) {
+ log.error(e, e);
+ }
+ UtilWaitThread.sleep(1000);
+ }
+ } else {
+ String value = StringUtil.join(entry.logSet, ";") + "|" + entry.tabletId;
+ Mutation m = new Mutation(entry.extent.getMetadataEntry());
+ m.put(Constants.METADATA_LOG_COLUMN_FAMILY, new Text(entry.server + "/" + entry.filename), new Value(value.getBytes(Constants.UTF8)));
+ update(credentials, zooLock, m);
+ }
+ }
-
++
+ public static LogEntry entryFromKeyValue(Key key, Value value) {
+ MetadataTable.LogEntry e = new MetadataTable.LogEntry();
+ e.extent = new KeyExtent(key.getRow(), EMPTY_TEXT);
+ String[] parts = key.getColumnQualifier().toString().split("/");
+ e.server = parts[0];
+ e.filename = parts[1];
+ parts = value.toString().split("\\|");
+ e.tabletId = Integer.parseInt(parts[1]);
+ e.logSet = Arrays.asList(parts[0].split(";"));
+ e.timestamp = key.getTimestamp();
+ return e;
+ }
+
+ public static Pair<List<LogEntry>,SortedMap<String,DataFileValue>> getFileAndLogEntries(TCredentials credentials, KeyExtent extent) throws KeeperException,
+ InterruptedException, IOException {
+ ArrayList<LogEntry> result = new ArrayList<LogEntry>();
+ TreeMap<String,DataFileValue> sizes = new TreeMap<String,DataFileValue>();
+
+ if (extent.isRootTablet()) {
+ getRootLogEntries(result);
+ FileSystem fs = TraceFileSystem.wrap(FileUtil.getFileSystem(CachedConfiguration.getInstance(), ServerConfiguration.getSiteConfiguration()));
+ FileStatus[] files = fs.listStatus(new Path(ServerConstants.getRootTabletDir()));
-
++
+ for (FileStatus fileStatus : files) {
+ if (fileStatus.getPath().toString().endsWith("_tmp")) {
+ continue;
+ }
+ DataFileValue dfv = new DataFileValue(0, 0);
+ sizes.put(Constants.ZROOT_TABLET + "/" + fileStatus.getPath().getName(), dfv);
+ }
-
++
+ } else {
+ Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ scanner.fetchColumnFamily(Constants.METADATA_LOG_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
+ scanner.setRange(extent.toMetadataRange());
-
++
+ for (Entry<Key,Value> entry : scanner) {
+ if (!entry.getKey().getRow().equals(extent.getMetadataEntry())) {
+ throw new RuntimeException("Unexpected row " + entry.getKey().getRow() + " expected " + extent.getMetadataEntry());
+ }
-
++
+ if (entry.getKey().getColumnFamily().equals(Constants.METADATA_LOG_COLUMN_FAMILY)) {
+ result.add(entryFromKeyValue(entry.getKey(), entry.getValue()));
+ } else if (entry.getKey().getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
+ DataFileValue dfv = new DataFileValue(entry.getValue().get());
+ sizes.put(entry.getKey().getColumnQualifier().toString(), dfv);
+ } else {
+ throw new RuntimeException("Unexpected col fam " + entry.getKey().getColumnFamily());
+ }
+ }
+ }
-
++
+ return new Pair<List<LogEntry>,SortedMap<String,DataFileValue>>(result, sizes);
+ }
+
+ public static List<LogEntry> getLogEntries(TCredentials credentials, KeyExtent extent) throws IOException, KeeperException, InterruptedException {
+ log.info("Scanning logging entries for " + extent);
+ ArrayList<LogEntry> result = new ArrayList<LogEntry>();
+ if (extent.equals(Constants.ROOT_TABLET_EXTENT)) {
+ log.info("Getting logs for root tablet from zookeeper");
+ getRootLogEntries(result);
+ } else {
+ log.info("Scanning metadata for logs used for tablet " + extent);
+ Scanner scanner = getTabletLogScanner(credentials, extent);
+ Text pattern = extent.getMetadataEntry();
+ for (Entry<Key,Value> entry : scanner) {
+ Text row = entry.getKey().getRow();
+ if (entry.getKey().getColumnFamily().equals(Constants.METADATA_LOG_COLUMN_FAMILY)) {
+ if (row.equals(pattern)) {
+ result.add(entryFromKeyValue(entry.getKey(), entry.getValue()));
+ }
+ }
+ }
+ }
-
++
+ Collections.sort(result, new Comparator<LogEntry>() {
+ @Override
+ public int compare(LogEntry o1, LogEntry o2) {
+ long diff = o1.timestamp - o2.timestamp;
+ if (diff < 0)
+ return -1;
+ if (diff > 0)
+ return 1;
+ return 0;
+ }
+ });
+ log.info("Returning logs " + result + " for extent " + extent);
+ return result;
+ }
-
++
+ private static void getRootLogEntries(ArrayList<LogEntry> result) throws KeeperException, InterruptedException, IOException {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ String root = getZookeeperLogLocation();
+ // there's a little race between getting the children and fetching
+ // the data. The log can be removed in between.
+ while (true) {
+ result.clear();
+ for (String child : zoo.getChildren(root)) {
+ LogEntry e = new LogEntry();
+ try {
+ e.fromBytes(zoo.getData(root + "/" + child, null));
+ result.add(e);
+ } catch (KeeperException.NoNodeException ex) {
+ continue;
+ }
+ }
+ break;
+ }
+ }
+
+ private static Scanner getTabletLogScanner(TCredentials credentials, KeyExtent extent) {
+ Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ scanner.fetchColumnFamily(Constants.METADATA_LOG_COLUMN_FAMILY);
+ Text start = extent.getMetadataEntry();
+ Key endKey = new Key(start, Constants.METADATA_LOG_COLUMN_FAMILY);
+ endKey = endKey.followingKey(PartialKey.ROW_COLFAM);
+ scanner.setRange(new Range(new Key(start), endKey));
+ return scanner;
+ }
-
++
+ static class LogEntryIterator implements Iterator<LogEntry> {
-
++
+ Iterator<LogEntry> rootTabletEntries = null;
+ Iterator<Entry<Key,Value>> metadataEntries = null;
+
+ LogEntryIterator(TCredentials creds) throws IOException, KeeperException, InterruptedException {
+ rootTabletEntries = getLogEntries(creds, Constants.ROOT_TABLET_EXTENT).iterator();
+ try {
+ Scanner scanner = HdfsZooInstance.getInstance().getConnector(creds.getPrincipal(), CredentialHelper.extractToken(creds))
+ .createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ scanner.fetchColumnFamily(Constants.METADATA_LOG_COLUMN_FAMILY);
+ metadataEntries = scanner.iterator();
+ } catch (Exception ex) {
+ throw new IOException(ex);
+ }
+ }
-
++
+ @Override
+ public boolean hasNext() {
+ return rootTabletEntries.hasNext() || metadataEntries.hasNext();
+ }
-
++
+ @Override
+ public LogEntry next() {
+ if (rootTabletEntries.hasNext()) {
+ return rootTabletEntries.next();
+ }
+ Entry<Key,Value> entry = metadataEntries.next();
+ return entryFromKeyValue(entry.getKey(), entry.getValue());
+ }
-
++
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ public static Iterator<LogEntry> getLogEntries(TCredentials creds) throws IOException, KeeperException, InterruptedException {
+ return new LogEntryIterator(creds);
+ }
-
++
+ public static void removeUnusedWALEntries(KeyExtent extent, List<LogEntry> logEntries, ZooLock zooLock) {
+ if (extent.isRootTablet()) {
+ for (LogEntry entry : logEntries) {
+ String root = getZookeeperLogLocation();
+ while (true) {
+ try {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ if (zoo.isLockHeld(zooLock.getLockID()))
+ zoo.recursiveDelete(root + "/" + entry.filename, NodeMissingPolicy.SKIP);
+ break;
+ } catch (Exception e) {
+ log.error(e, e);
+ }
+ UtilWaitThread.sleep(1000);
+ }
+ }
+ } else {
+ Mutation m = new Mutation(extent.getMetadataEntry());
+ for (LogEntry entry : logEntries) {
+ m.putDelete(Constants.METADATA_LOG_COLUMN_FAMILY, new Text(entry.server + "/" + entry.filename));
+ }
+ update(SecurityConstants.getSystemCredentials(), zooLock, m);
+ }
+ }
-
++
+ private static void getFiles(Set<String> files, Map<Key,Value> tablet, String srcTableId) {
+ for (Entry<Key,Value> entry : tablet.entrySet()) {
+ if (entry.getKey().getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
+ String cf = entry.getKey().getColumnQualifier().toString();
+ if (srcTableId != null && !cf.startsWith("../"))
+ cf = "../" + srcTableId + entry.getKey().getColumnQualifier();
+ files.add(cf);
+ }
+ }
+ }
-
++
+ private static Mutation createCloneMutation(String srcTableId, String tableId, Map<Key,Value> tablet) {
-
++
+ KeyExtent ke = new KeyExtent(tablet.keySet().iterator().next().getRow(), (Text) null);
+ Mutation m = new Mutation(KeyExtent.getMetadataEntry(new Text(tableId), ke.getEndRow()));
-
++
+ for (Entry<Key,Value> entry : tablet.entrySet()) {
+ if (entry.getKey().getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
+ String cf = entry.getKey().getColumnQualifier().toString();
+ if (!cf.startsWith("../"))
+ cf = "../" + srcTableId + entry.getKey().getColumnQualifier();
+ m.put(entry.getKey().getColumnFamily(), new Text(cf), entry.getValue());
+ } else if (entry.getKey().getColumnFamily().equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY)) {
+ m.put(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY, entry.getKey().getColumnQualifier(), entry.getValue());
+ } else if (entry.getKey().getColumnFamily().equals(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY)) {
+ // skip
+ } else {
+ m.put(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier(), entry.getValue());
+ }
+ }
+ return m;
+ }
-
++
+ private static Scanner createCloneScanner(String tableId, Connector conn) throws TableNotFoundException {
+ Scanner mscanner = new IsolatedScanner(conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS));
+ mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+ mscanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
+ mscanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
+ mscanner.fetchColumnFamily(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY);
+ mscanner.fetchColumnFamily(Constants.METADATA_CLONED_COLUMN_FAMILY);
+ Constants.METADATA_PREV_ROW_COLUMN.fetch(mscanner);
+ Constants.METADATA_TIME_COLUMN.fetch(mscanner);
+ return mscanner;
+ }
-
++
+ static void initializeClone(String srcTableId, String tableId, Connector conn, BatchWriter bw) throws TableNotFoundException, MutationsRejectedException {
+ TabletIterator ti = new TabletIterator(createCloneScanner(srcTableId, conn), new KeyExtent(new Text(srcTableId), null, null).toMetadataRange(), true, true);
-
++
+ if (!ti.hasNext())
+ throw new RuntimeException(" table deleted during clone? srcTableId = " + srcTableId);
-
++
+ while (ti.hasNext())
+ bw.addMutation(createCloneMutation(srcTableId, tableId, ti.next()));
-
++
+ bw.flush();
+ }
-
++
+ static int compareEndRows(Text endRow1, Text endRow2) {
+ return new KeyExtent(new Text("0"), endRow1, null).compareTo(new KeyExtent(new Text("0"), endRow2, null));
+ }
-
++
+ static int checkClone(String srcTableId, String tableId, Connector conn, BatchWriter bw) throws TableNotFoundException, MutationsRejectedException {
+ TabletIterator srcIter = new TabletIterator(createCloneScanner(srcTableId, conn), new KeyExtent(new Text(srcTableId), null, null).toMetadataRange(), true,
+ true);
+ TabletIterator cloneIter = new TabletIterator(createCloneScanner(tableId, conn), new KeyExtent(new Text(tableId), null, null).toMetadataRange(), true, true);
-
++
+ if (!cloneIter.hasNext() || !srcIter.hasNext())
+ throw new RuntimeException(" table deleted during clone? srcTableId = " + srcTableId + " tableId=" + tableId);
-
++
+ int rewrites = 0;
-
++
+ while (cloneIter.hasNext()) {
+ Map<Key,Value> cloneTablet = cloneIter.next();
+ Text cloneEndRow = new KeyExtent(cloneTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
+ HashSet<String> cloneFiles = new HashSet<String>();
-
++
+ boolean cloneSuccessful = false;
+ for (Entry<Key,Value> entry : cloneTablet.entrySet()) {
+ if (entry.getKey().getColumnFamily().equals(Constants.METADATA_CLONED_COLUMN_FAMILY)) {
+ cloneSuccessful = true;
+ break;
+ }
+ }
-
++
+ if (!cloneSuccessful)
+ getFiles(cloneFiles, cloneTablet, null);
-
++
+ List<Map<Key,Value>> srcTablets = new ArrayList<Map<Key,Value>>();
+ Map<Key,Value> srcTablet = srcIter.next();
+ srcTablets.add(srcTablet);
-
++
+ Text srcEndRow = new KeyExtent(srcTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
-
++
+ int cmp = compareEndRows(cloneEndRow, srcEndRow);
+ if (cmp < 0)
+ throw new TabletIterator.TabletDeletedException("Tablets deleted from src during clone : " + cloneEndRow + " " + srcEndRow);
-
++
+ HashSet<String> srcFiles = new HashSet<String>();
+ if (!cloneSuccessful)
+ getFiles(srcFiles, srcTablet, srcTableId);
-
++
+ while (cmp > 0) {
+ srcTablet = srcIter.next();
+ srcTablets.add(srcTablet);
+ srcEndRow = new KeyExtent(srcTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
+ cmp = compareEndRows(cloneEndRow, srcEndRow);
+ if (cmp < 0)
+ throw new TabletIterator.TabletDeletedException("Tablets deleted from src during clone : " + cloneEndRow + " " + srcEndRow);
-
++
+ if (!cloneSuccessful)
+ getFiles(srcFiles, srcTablet, srcTableId);
+ }
-
++
+ if (cloneSuccessful)
+ continue;
-
++
+ if (!srcFiles.containsAll(cloneFiles)) {
+ // delete existing cloned tablet entry
+ Mutation m = new Mutation(cloneTablet.keySet().iterator().next().getRow());
-
++
+ for (Entry<Key,Value> entry : cloneTablet.entrySet()) {
+ Key k = entry.getKey();
+ m.putDelete(k.getColumnFamily(), k.getColumnQualifier(), k.getTimestamp());
+ }
-
++
+ bw.addMutation(m);
-
++
+ for (Map<Key,Value> st : srcTablets)
+ bw.addMutation(createCloneMutation(srcTableId, tableId, st));
-
++
+ rewrites++;
+ } else {
+ // write out marker that this tablet was successfully cloned
+ Mutation m = new Mutation(cloneTablet.keySet().iterator().next().getRow());
+ m.put(Constants.METADATA_CLONED_COLUMN_FAMILY, new Text(""), new Value("OK".getBytes(Constants.UTF8)));
+ bw.addMutation(m);
+ }
+ }
-
++
+ bw.flush();
+ return rewrites;
+ }
-
++
+ public static void cloneTable(Instance instance, String srcTableId, String tableId) throws Exception {
+
+ Connector conn = instance.getConnector(SecurityConstants.SYSTEM_PRINCIPAL, SecurityConstants.getSystemToken());
+ BatchWriter bw = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig());
+
+ while (true) {
-
++
+ try {
+ initializeClone(srcTableId, tableId, conn, bw);
-
++
+ // the following loop looks changes in the file that occurred during the copy.. if files were dereferenced then they could have been GCed
-
++
+ while (true) {
+ int rewrites = checkClone(srcTableId, tableId, conn, bw);
-
++
+ if (rewrites == 0)
+ break;
+ }
-
++
+ bw.flush();
+ break;
-
++
+ } catch (TabletIterator.TabletDeletedException tde) {
+ // tablets were merged in the src table
+ bw.flush();
-
++
+ // delete what we have cloned and try again
+ deleteTable(tableId, false, SecurityConstants.getSystemCredentials(), null);
-
++
+ log.debug("Tablets merged in table " + srcTableId + " while attempting to clone, trying again");
-
++
+ UtilWaitThread.sleep(100);
+ }
+ }
-
++
+ // delete the clone markers and create directory entries
+ Scanner mscanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+ mscanner.fetchColumnFamily(Constants.METADATA_CLONED_COLUMN_FAMILY);
-
++
+ int dirCount = 0;
-
++
+ for (Entry<Key,Value> entry : mscanner) {
+ Key k = entry.getKey();
+ Mutation m = new Mutation(k.getRow());
+ m.putDelete(k.getColumnFamily(), k.getColumnQualifier());
+ Constants.METADATA_DIRECTORY_COLUMN.put(m, new Value(FastFormat.toZeroPaddedString(dirCount++, 8, 16, "/c-".getBytes(Constants.UTF8))));
+ bw.addMutation(m);
+ }
-
++
+ bw.close();
-
++
+ }
-
++
+ public static void chopped(KeyExtent extent, ZooLock zooLock) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
+ Constants.METADATA_CHOPPED_COLUMN.put(m, new Value("chopped".getBytes(Constants.UTF8)));
+ update(SecurityConstants.getSystemCredentials(), zooLock, m);
+ }
-
++
+ public static void removeBulkLoadEntries(Connector conn, String tableId, long tid) throws Exception {
+ Scanner mscanner = new IsolatedScanner(conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS));
+ mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+ mscanner.fetchColumnFamily(Constants.METADATA_BULKFILE_COLUMN_FAMILY);
+ BatchWriter bw = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig());
+ for (Entry<Key,Value> entry : mscanner) {
+ log.debug("Looking at entry " + entry + " with tid " + tid);
+ if (Long.parseLong(entry.getValue().toString()) == tid) {
+ log.debug("deleting entry " + entry);
+ Mutation m = new Mutation(entry.getKey().getRow());
+ m.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier());
+ bw.addMutation(m);
+ }
+ }
+ bw.close();
+ }
-
++
+ public static List<String> getBulkFilesLoaded(Connector conn, KeyExtent extent, long tid) {
+ List<String> result = new ArrayList<String>();
+ try {
+ Scanner mscanner = new IsolatedScanner(conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS));
+ mscanner.setRange(extent.toMetadataRange());
+ mscanner.fetchColumnFamily(Constants.METADATA_BULKFILE_COLUMN_FAMILY);
+ for (Entry<Key,Value> entry : mscanner) {
+ if (Long.parseLong(entry.getValue().toString()) == tid) {
+ result.add(entry.getKey().getColumnQualifier().toString());
+ }
+ }
+ return result;
+ } catch (TableNotFoundException ex) {
+ // unlikely
+ throw new RuntimeException("Onos! teh metadata table has vanished!!");
+ }
+ }
+
+ public static Map<String,Long> getBulkFilesLoaded(TCredentials credentials, KeyExtent extent) {
+ return getBulkFilesLoaded(credentials, extent.getMetadataEntry());
+ }
+
+ public static Map<String,Long> getBulkFilesLoaded(TCredentials credentials, Text metadataRow) {
+
+ Map<String,Long> ret = new HashMap<String,Long>();
-
++
+ Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ scanner.setRange(new Range(metadataRow));
+ scanner.fetchColumnFamily(Constants.METADATA_BULKFILE_COLUMN_FAMILY);
+ for (Entry<Key,Value> entry : scanner) {
+ String file = entry.getKey().getColumnQualifier().toString();
+ Long tid = Long.parseLong(entry.getValue().toString());
-
++
+ ret.put(file, tid);
+ }
+ return ret;
+ }
+
+ public static void addBulkLoadInProgressFlag(String path) {
-
++
+ Mutation m = new Mutation(Constants.METADATA_BLIP_FLAG_PREFIX + path);
+ m.put(EMPTY_TEXT, EMPTY_TEXT, new Value(new byte[] {}));
-
++
+ update(SecurityConstants.getSystemCredentials(), m);
+ }
-
++
+ public static void removeBulkLoadInProgressFlag(String path) {
-
++
+ Mutation m = new Mutation(Constants.METADATA_BLIP_FLAG_PREFIX + path);
+ m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
-
++
+ update(SecurityConstants.getSystemCredentials(), m);
+ }
+
+ /**
+ * During an upgrade from Accumulo 1.4 -> 1.5, we need to move deletion requests for files under the !METADATA table to the root tablet.
+ */
+ public static void moveMetaDeleteMarkers(Instance instance, TCredentials creds) {
+ // move delete markers from the normal delete keyspace to the root tablet delete keyspace if the files are for the !METADATA table
+ Scanner scanner = new ScannerImpl(instance, creds, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
+ scanner.setRange(new Range(Constants.METADATA_DELETES_KEYSPACE));
+ for (Entry<Key,Value> entry : scanner) {
+ String row = entry.getKey().getRow().toString();
+ if (row.startsWith(Constants.METADATA_DELETE_FLAG_PREFIX + "/" + Constants.METADATA_TABLE_ID)) {
+ String filename = row.substring(Constants.METADATA_DELETE_FLAG_PREFIX.length());
+ // add the new entry first
+ log.info("Moving " + filename + " marker to the root tablet");
+ Mutation m = new Mutation(Constants.METADATA_DELETE_FLAG_FOR_METADATA_PREFIX + filename);
+ m.put(new byte[]{}, new byte[]{}, new byte[]{});
+ update(creds, m);
+ // remove the old entry
+ m = new Mutation(entry.getKey().getRow());
+ m.putDelete(new byte[]{}, new byte[]{});
+ update(creds, m);
+ } else {
+ break;
+ }
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
index 8985b27,0000000..8e41db7
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
@@@ -1,127 -1,0 +1,123 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.Stack;
+
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.DefaultHandler;
+
+import com.beust.jcommander.Parameter;
+
+public class RestoreZookeeper {
+
+ private static class Restore extends DefaultHandler {
+ IZooReaderWriter zk = null;
+ Stack<String> cwd = new Stack<String>();
+ boolean overwrite = false;
+
+ Restore(IZooReaderWriter zk, boolean overwrite) {
+ this.zk = zk;
+ this.overwrite = overwrite;
+ }
+
+ @Override
+ public void startElement(String uri, String localName, String name, Attributes attributes) throws SAXException {
+ if ("node".equals(name)) {
+ String child = attributes.getValue("name");
+ if (child == null)
+ throw new RuntimeException("name attribute not set");
+ String encoding = attributes.getValue("encoding");
+ String value = attributes.getValue("value");
+ if (value == null)
+ value = "";
+ String path = cwd.lastElement() + "/" + child;
+ create(path, value, encoding);
+ cwd.push(path);
+ } else if ("dump".equals(name)) {
+ String root = attributes.getValue("root");
+ if (root.equals("/"))
+ cwd.push("");
+ else
+ cwd.push(root);
+ create(root, "", "utf-8");
+ }
+ }
+
+ @Override
+ public void endElement(String uri, String localName, String name) throws SAXException {
+ cwd.pop();
+ }
+
+ private void create(String path, String value, String encoding) {
+ byte[] data = value.getBytes(Constants.UTF8);
+ if ("base64".equals(encoding))
+ data = Base64.decodeBase64(data);
+ try {
+ try {
+ zk.putPersistentData(path, data, overwrite ? NodeExistsPolicy.OVERWRITE : NodeExistsPolicy.FAIL);
+ } catch (KeeperException e) {
+ if (e.code().equals(KeeperException.Code.NODEEXISTS))
+ throw new RuntimeException(path + " exists. Remove it first.");
+ throw e;
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static class Opts extends Help {
+ @Parameter(names={"-z", "--keepers"})
+ String keepers = "localhost:2181";
+ @Parameter(names="--overwrite")
+ boolean overwrite = false;
+ @Parameter(names="--file")
+ String file;
+ }
+
- /**
- * @param args
- * @throws Exception
- */
+ public static void main(String[] args) throws Exception {
+ Logger.getRootLogger().setLevel(Level.WARN);
+ Opts opts = new Opts();
+ opts.parseArgs(RestoreZookeeper.class.getName(), args);
+
+ InputStream in = System.in;
+ if (opts.file != null) {
+ in = new FileInputStream(opts.file);
+ }
+
+ SAXParserFactory factory = SAXParserFactory.newInstance();
+ SAXParser parser = factory.newSAXParser();
+ parser.parse(in, new Restore(ZooReaderWriter.getInstance(), opts.overwrite));
+ in.close();
+ }
+}
[16/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
index 156fa3a,0000000..2a1372c
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperationsImpl.java
@@@ -1,242 -1,0 +1,209 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.admin;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.ClientExec;
+import org.apache.accumulo.core.client.impl.ClientExecReturn;
+import org.apache.accumulo.core.client.impl.MasterClient;
+import org.apache.accumulo.core.client.impl.ServerClient;
+import org.apache.accumulo.core.client.impl.thrift.ClientService;
+import org.apache.accumulo.core.client.impl.thrift.ConfigurationType;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+
+/**
+ * Provides a class for administering the accumulo instance
+ */
+public class InstanceOperationsImpl implements InstanceOperations {
+ private Instance instance;
+ private TCredentials credentials;
+
+ /**
+ * @param instance
+ * the connection information for this instance
+ * @param credentials
+ * the Credential, containing principal and Authentication Token
+ */
+ public InstanceOperationsImpl(Instance instance, TCredentials credentials) {
+ ArgumentChecker.notNull(instance, credentials);
+ this.instance = instance;
+ this.credentials = credentials;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#setProperty(java.lang.String, java.lang.String)
- */
+ @Override
+ public void setProperty(final String property, final String value) throws AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(property, value);
+ MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() {
+ @Override
+ public void execute(MasterClientService.Client client) throws Exception {
+ client.setSystemProperty(Tracer.traceInfo(), credentials, property, value);
+ }
+ });
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#removeProperty(java.lang.String)
- */
+ @Override
+ public void removeProperty(final String property) throws AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(property);
+ MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() {
+ @Override
+ public void execute(MasterClientService.Client client) throws Exception {
+ client.removeSystemProperty(Tracer.traceInfo(), credentials, property);
+ }
+ });
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getSystemConfiguration()
- */
+ @Override
+ public Map<String,String> getSystemConfiguration() throws AccumuloException, AccumuloSecurityException {
+ return ServerClient.execute(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() {
+ @Override
+ public Map<String,String> execute(ClientService.Client client) throws Exception {
+ return client.getConfiguration(Tracer.traceInfo(), credentials, ConfigurationType.CURRENT);
+ }
+ });
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getSiteConfiguration()
- */
+ @Override
+ public Map<String,String> getSiteConfiguration() throws AccumuloException, AccumuloSecurityException {
+ return ServerClient.execute(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() {
+ @Override
+ public Map<String,String> execute(ClientService.Client client) throws Exception {
+ return client.getConfiguration(Tracer.traceInfo(), credentials, ConfigurationType.SITE);
+ }
+ });
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getTabletServers()
- */
-
+ @Override
+ public List<String> getTabletServers() {
+ ZooCache cache = ZooCache.getInstance(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
+ String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
+ List<String> results = new ArrayList<String>();
+ for (String candidate : cache.getChildren(path)) {
+ List<String> children = cache.getChildren(path + "/" + candidate);
+ if (children != null && children.size() > 0) {
+ List<String> copy = new ArrayList<String>(children);
+ Collections.sort(copy);
+ byte[] data = cache.get(path + "/" + candidate + "/" + copy.get(0));
+ if (data != null && !"master".equals(new String(data, Constants.UTF8))) {
+ results.add(candidate);
+ }
+ }
+ }
+ return results;
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getActiveScans(java.lang.String)
- */
-
+ @Override
+ public List<ActiveScan> getActiveScans(String tserver) throws AccumuloException, AccumuloSecurityException {
+ Client client = null;
+ try {
+ client = ThriftUtil.getTServerClient(tserver, instance.getConfiguration());
+
+ List<ActiveScan> as = new ArrayList<ActiveScan>();
+ for (org.apache.accumulo.core.tabletserver.thrift.ActiveScan activeScan : client.getActiveScans(Tracer.traceInfo(), credentials)) {
+ try {
+ as.add(new ActiveScan(instance, activeScan));
+ } catch (TableNotFoundException e) {
+ throw new AccumuloException(e);
+ }
+ }
+ return as;
+ } catch (TTransportException e) {
+ throw new AccumuloException(e);
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (TException e) {
+ throw new AccumuloException(e);
+ } finally {
+ if (client != null)
+ ThriftUtil.returnClient(client);
+ }
+ }
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#testClassLoad(java.lang.String, java.lang.String)
- */
+ @Override
+ public boolean testClassLoad(final String className, final String asTypeName) throws AccumuloException, AccumuloSecurityException {
+ return ServerClient.execute(instance, new ClientExecReturn<Boolean,ClientService.Client>() {
+ @Override
+ public Boolean execute(ClientService.Client client) throws Exception {
+ return client.checkClass(Tracer.traceInfo(), credentials, className, asTypeName);
+ }
+ });
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getActiveCompactions(java.lang.String)
- */
+ @Override
+ public List<ActiveCompaction> getActiveCompactions(String tserver) throws AccumuloException, AccumuloSecurityException {
+ Client client = null;
+ try {
+ client = ThriftUtil.getTServerClient(tserver, instance.getConfiguration());
+
+ List<ActiveCompaction> as = new ArrayList<ActiveCompaction>();
+ for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction activeCompaction : client.getActiveCompactions(Tracer.traceInfo(), credentials)) {
+ as.add(new ActiveCompaction(instance, activeCompaction));
+ }
+ return as;
+ } catch (TTransportException e) {
+ throw new AccumuloException(e);
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (TException e) {
+ throw new AccumuloException(e);
+ } finally {
+ if (client != null)
+ ThriftUtil.returnClient(client);
+ }
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#ping(java.lang.String)
- */
+ @Override
+ public void ping(String tserver) throws AccumuloException {
+ TTransport transport = null;
+ try {
+ transport = ThriftUtil.createTransport(tserver, instance.getConfiguration().getPort(Property.TSERV_CLIENTPORT), instance.getConfiguration());
+ TabletClientService.Client client = ThriftUtil.createClient(new TabletClientService.Client.Factory(), transport);
+ client.getTabletServerStatus(Tracer.traceInfo(), credentials);
+ } catch (TTransportException e) {
+ throw new AccumuloException(e);
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloException(e);
+ } catch (TException e) {
+ throw new AccumuloException(e);
+ } finally {
+ if (transport != null) {
+ transport.close();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
index 2521c96,0000000..0823656
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
@@@ -1,719 -1,0 +1,687 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.admin;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedSet;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Provides a class for administering tables
+ *
+ */
+
+public interface TableOperations {
+
+ /**
+ * Retrieve a list of tables in Accumulo.
+ *
+ * @return List of tables in accumulo
+ */
+ public SortedSet<String> list();
+
+ /**
+ * A method to check if a table exists in Accumulo.
+ *
+ * @param tableName
+ * the name of the table
+ * @return true if the table exists
+ */
+ public boolean exists(String tableName);
+
+ /**
+ * Create a table with no special configuration
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableExistsException
+ * if the table already exists
+ */
+ public void create(String tableName) throws AccumuloException, AccumuloSecurityException, TableExistsException;
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param limitVersion
+ * Enables/disables the versioning iterator, which will limit the number of Key versions kept.
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableExistsException
+ * if the table already exists
+ */
+ public void create(String tableName, boolean limitVersion) throws AccumuloException, AccumuloSecurityException, TableExistsException;
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param versioningIter
+ * Enables/disables the versioning iterator, which will limit the number of Key versions kept.
+ * @param timeType
+ * specifies logical or real-time based time recording for entries in the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableExistsException
+ * if the table already exists
+ */
+ public void create(String tableName, boolean versioningIter, TimeType timeType) throws AccumuloException, AccumuloSecurityException, TableExistsException;
+
+ /**
+ * Imports a table exported via exportTable and copied via hadoop distcp.
+ *
+ * @param tableName
+ * Name of a table to create and import into.
+ * @param importDir
+ * Directory that contains the files copied by distcp from exportTable
- * @throws TableExistsException
- * @throws AccumuloException
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public void importTable(String tableName, String importDir) throws TableExistsException, AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Exports a table. The tables data is not exported, just table metadata and a list of files to distcp. The table being exported must be offline and stay
+ * offline for the duration of distcp. To avoid losing access to a table it can be cloned and the clone taken offline for export.
+ *
+ * <p>
+ * See docs/examples/README.export
+ *
+ * @param tableName
+ * Name of the table to export.
+ * @param exportDir
+ * An empty directory in HDFS where files containing table metadata and list of files to distcp will be placed.
- * @throws TableNotFoundException
- * @throws AccumuloException
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public void exportTable(String tableName, String exportDir) throws TableNotFoundException, AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Ensures that tablets are split along a set of keys.
+ * <p>
+ * Note that while the documentation for Text specifies that its bytestream should be UTF-8, the encoding is not enforced by operations that work with byte arrays.
+ * <p>
+ * For example, you can create 256 evenly-sliced splits via the following code sample even though the given byte sequences are not valid UTF-8.
+ * <pre>
+ * {@code
+ * TableOperations tableOps = connector.tableOperations();
+ * TreeSet<Text> splits = new TreeSet<Text>();
+ * for (int i = 0; i < 256; i++) {
+ * byte[] bytes = { (byte) i };
+ * splits.add(new Text(bytes));
+ * }
+ * tableOps.addSplits(TABLE_NAME, splits);
+ * }
+ * </pre>
+ *
+ * @param tableName
+ * the name of the table
+ * @param partitionKeys
+ * a sorted set of row key values to pre-split the table on
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ public void addSplits(String tableName, SortedSet<Text> partitionKeys) throws TableNotFoundException, AccumuloException, AccumuloSecurityException;
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @return the split points (end-row names) for the table's current split profile
+ * @throws TableNotFoundException
+ * if the table does not exist
+ * @deprecated since 1.5.0; use {@link #listSplits(String)} instead.
+ */
+ @Deprecated
+ public Collection<Text> getSplits(String tableName) throws TableNotFoundException;
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @return the split points (end-row names) for the table's current split profile
+ * @throws TableNotFoundException
+ * if the table does not exist
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @since 1.5.0
+ */
+ public Collection<Text> listSplits(String tableName) throws TableNotFoundException, AccumuloSecurityException, AccumuloException;
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param maxSplits
+ * specifies the maximum number of splits to return
+ * @return the split points (end-row names) for the table's current split profile, grouped into fewer splits so as not to exceed maxSplits
- * @throws TableNotFoundException
+ * @deprecated since 1.5.0; use {@link #listSplits(String, int)} instead.
+ */
+ @Deprecated
+ public Collection<Text> getSplits(String tableName, int maxSplits) throws TableNotFoundException;
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param maxSplits
+ * specifies the maximum number of splits to return
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @return the split points (end-row names) for the table's current split profile, grouped into fewer splits so as not to exceed maxSplits
- * @throws TableNotFoundException
+ * @since 1.5.0
+ */
+ public Collection<Text> listSplits(String tableName, int maxSplits) throws TableNotFoundException, AccumuloSecurityException, AccumuloException;
+
+ /**
+ * Finds the max row within a given range. To find the max row in a table, pass null for start and end row.
+ *
- * @param tableName
+ * @param auths
+ * find the max row that can seen with these auths
+ * @param startRow
+ * row to start looking at, null means -Infinity
+ * @param startInclusive
+ * determines if the start row is included
+ * @param endRow
+ * row to stop looking at, null means Infinity
+ * @param endInclusive
+ * determines if the end row is included
+ *
+ * @return The max row in the range, or null if there is no visible data in the range.
- *
- * @throws AccumuloSecurityException
- * @throws AccumuloException
- * @throws TableNotFoundException
+ */
+ public Text getMaxRow(String tableName, Authorizations auths, Text startRow, boolean startInclusive, Text endRow, boolean endInclusive)
+ throws TableNotFoundException, AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Merge tablets between (start, end]
+ *
+ * @param tableName
+ * the table to merge
+ * @param start
+ * first tablet to be merged contains the row after this row, null means the first tablet
+ * @param end
+ * last tablet to be merged contains this row, null means the last tablet
+ */
+ public void merge(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
+ /**
+ * Delete rows between (start, end]
+ *
+ * @param tableName
+ * the table to merge
+ * @param start
+ * delete rows after this, null means the first row of the table
+ * @param end
+ * last row to be deleted, inclusive, null means the last row of the table
+ */
+ public void deleteRows(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
+ /**
+ * Starts a full major compaction of the tablets in the range (start, end]. The compaction is preformed even for tablets that have only one file.
+ *
+ * @param tableName
+ * the table to compact
+ * @param start
+ * first tablet to be compacted contains the row after this row, null means the first tablet in table
+ * @param end
+ * last tablet to be merged contains this row, null means the last tablet in table
+ * @param flush
+ * when true, table memory is flushed before compaction starts
+ * @param wait
+ * when true, the call will not return until compactions are finished
+ */
+ public void compact(String tableName, Text start, Text end, boolean flush, boolean wait) throws AccumuloSecurityException, TableNotFoundException,
+ AccumuloException;
+
+ /**
+ * Starts a full major compaction of the tablets in the range (start, end]. The compaction is preformed even for tablets that have only one file.
+ *
+ * @param tableName
+ * the table to compact
+ * @param start
+ * first tablet to be compacted contains the row after this row, null means the first tablet in table
+ * @param end
+ * last tablet to be merged contains this row, null means the last tablet in table
+ * @param iterators
+ * A set of iterators that will be applied to each tablet compacted
+ * @param flush
+ * when true, table memory is flushed before compaction starts
+ * @param wait
+ * when true, the call will not return until compactions are finished
+ * @since 1.5.0
+ */
+ public void compact(String tableName, Text start, Text end, List<IteratorSetting> iterators, boolean flush, boolean wait) throws AccumuloSecurityException,
+ TableNotFoundException, AccumuloException;
+
+ /**
+ * Cancels a user initiated major compaction of a table initiated with {@link #compact(String, Text, Text, boolean, boolean)} or
+ * {@link #compact(String, Text, Text, List, boolean, boolean)}. Compactions of tablets that are currently running may finish, but new compactions of tablets
+ * will not start.
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws TableNotFoundException
+ * if the table does not exist
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @since 1.5.0
+ */
+ public void cancelCompaction(String tableName) throws AccumuloSecurityException, TableNotFoundException, AccumuloException;
+
+ /**
+ * Delete a table
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ public void delete(String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
+ /**
+ * Clone a table from an existing table. The cloned table will have the same data as the source table it was created from. After cloning, the two tables can
+ * mutate independently. Initially the cloned table should not use any extra space, however as the source table and cloned table major compact extra space
+ * will be used by the clone.
+ *
+ * Initially the cloned table is only readable and writable by the user who created it.
+ *
+ * @param srcTableName
+ * the table to clone
+ * @param newTableName
+ * the name of the clone
+ * @param flush
+ * determines if memory is flushed in the source table before cloning.
+ * @param propertiesToSet
+ * the sources tables properties are copied, this allows overriding of those properties
+ * @param propertiesToExclude
+ * do not copy these properties from the source table, just revert to system defaults
+ */
+
+ public void clone(String srcTableName, String newTableName, boolean flush, Map<String,String> propertiesToSet, Set<String> propertiesToExclude)
+ throws AccumuloException, AccumuloSecurityException, TableNotFoundException, TableExistsException;
+
+ /**
+ * Rename a table
+ *
+ * @param oldTableName
+ * the old table name
+ * @param newTableName
+ * the new table name
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the old table name does not exist
+ * @throws TableExistsException
+ * if the new table name already exists
+ */
+ public void rename(String oldTableName, String newTableName) throws AccumuloSecurityException, TableNotFoundException, AccumuloException,
+ TableExistsException;
+
+ /**
+ * Initiate a flush of a table's data that is in memory
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ *
+ * @deprecated As of release 1.4, replaced by {@link #flush(String, Text, Text, boolean)}
+ */
+ @Deprecated
+ public void flush(String tableName) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Flush a table's data that is currently in memory.
+ *
+ * @param tableName
+ * the name of the table
+ * @param wait
+ * if true the call will not return until all data present in memory when the call was is flushed if false will initiate a flush of data in memory,
+ * but will not wait for it to complete
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
- * @throws TableNotFoundException
+ */
+ public void flush(String tableName, Text start, Text end, boolean wait) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
+ /**
+ * Sets a property on a table. Note that it may take a short period of time (a second) to propagate the change everywhere.
+ *
+ * @param tableName
+ * the name of the table
+ * @param property
+ * the name of a per-table property
+ * @param value
+ * the value to set a per-table property to
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ public void setProperty(String tableName, String property, String value) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Removes a property from a table. Note that it may take a short period of time (a second) to propagate the change everywhere.
+ *
+ * @param tableName
+ * the name of the table
+ * @param property
+ * the name of a per-table property
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ public void removeProperty(String tableName, String property) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Gets properties of a table. Note that recently changed properties may not be available immediately.
+ *
+ * @param tableName
+ * the name of the table
+ * @return all properties visible by this table (system and per-table properties). Note that recently changed
+ * properties may not be visible immediately.
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ public Iterable<Entry<String,String>> getProperties(String tableName) throws AccumuloException, TableNotFoundException;
+
+ /**
+ * Sets a table's locality groups. A table's locality groups can be changed at any time.
+ *
+ * @param tableName
+ * the name of the table
+ * @param groups
+ * mapping of locality group names to column families in the locality group
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ public void setLocalityGroups(String tableName, Map<String,Set<Text>> groups) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
+ /**
+ *
+ * Gets the locality groups currently set for a table.
+ *
+ * @param tableName
+ * the name of the table
+ * @return mapping of locality group names to column families in the locality group
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ public Map<String,Set<Text>> getLocalityGroups(String tableName) throws AccumuloException, TableNotFoundException;
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param range
+ * a range to split
+ * @param maxSplits
+ * the maximum number of splits
+ * @return the range, split into smaller ranges that fall on boundaries of the table's split points as evenly as possible
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ public Set<Range> splitRangeByTablets(String tableName, Range range, int maxSplits) throws AccumuloException, AccumuloSecurityException,
+ TableNotFoundException;
+
+ /**
+ * Bulk import all the files in a directory into a table.
+ *
+ * @param tableName
+ * the name of the table
+ * @param dir
+ * the HDFS directory to find files for importing
+ * @param failureDir
+ * the HDFS directory to place files that failed to be imported, must exist and be empty
+ * @param setTime
+ * override the time values in the input files, and use the current time for all mutations
+ * @throws IOException
+ * when there is an error reading/writing to HDFS
+ * @throws AccumuloException
+ * when there is a general accumulo error
+ * @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
+ * @throws TableNotFoundException
+ * when the table no longer exists
+ *
+ */
+ public void importDirectory(String tableName, String dir, String failureDir, boolean setTime) throws TableNotFoundException, IOException, AccumuloException,
+ AccumuloSecurityException;
+
+ /**
+ *
+ * @param tableName
+ * the table to take offline
+ * @throws AccumuloException
+ * when there is a general accumulo error
+ * @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
- * @throws TableNotFoundException
+ */
+ public void offline(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
+
+ /**
+ *
+ * @param tableName
+ * the table to take online
+ * @throws AccumuloException
+ * when there is a general accumulo error
+ * @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
- * @throws TableNotFoundException
+ */
+ public void online(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
+
+ /**
+ * Clears the tablet locator cache for a specified table
+ *
+ * @param tableName
+ * the name of the table
+ * @throws TableNotFoundException
+ * if table does not exist
+ */
+ public void clearLocatorCache(String tableName) throws TableNotFoundException;
+
+ /**
+ * Get a mapping of table name to internal table id.
+ *
+ * @return the map from table name to internal table id
+ */
+ public Map<String,String> tableIdMap();
+
+ /**
+ * Add an iterator to a table on all scopes.
+ *
+ * @param tableName
+ * the name of the table
+ * @param setting
+ * object specifying the properties of the iterator
+ * @throws AccumuloSecurityException
+ * thrown if the user does not have the ability to set properties on the table
- * @throws AccumuloException
+ * @throws TableNotFoundException
+ * throw if the table no longer exists
+ * @throws IllegalArgumentException
+ * if the setting conflicts with any existing iterators
+ */
+ public void attachIterator(String tableName, IteratorSetting setting) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
+
+ /**
+ * Add an iterator to a table on the given scopes.
+ *
+ * @param tableName
+ * the name of the table
+ * @param setting
+ * object specifying the properties of the iterator
+ * @throws AccumuloSecurityException
+ * thrown if the user does not have the ability to set properties on the table
- * @throws AccumuloException
+ * @throws TableNotFoundException
+ * throw if the table no longer exists
+ * @throws IllegalArgumentException
+ * if the setting conflicts with any existing iterators
+ */
+ public void attachIterator(String tableName, IteratorSetting setting, EnumSet<IteratorScope> scopes) throws AccumuloSecurityException, AccumuloException,
+ TableNotFoundException;
+
+ /**
+ * Remove an iterator from a table by name.
+ *
+ * @param tableName
+ * the name of the table
+ * @param name
+ * the name of the iterator
+ * @param scopes
+ * the scopes of the iterator
+ * @throws AccumuloSecurityException
+ * thrown if the user does not have the ability to set properties on the table
- * @throws AccumuloException
+ * @throws TableNotFoundException
+ * throw if the table no longer exists
+ */
+ public void removeIterator(String tableName, String name, EnumSet<IteratorScope> scopes) throws AccumuloSecurityException, AccumuloException,
+ TableNotFoundException;
+
+ /**
+ * Get the settings for an iterator.
+ *
+ * @param tableName
+ * the name of the table
+ * @param name
+ * the name of the iterator
+ * @param scope
+ * the scope of the iterator
+ * @return the settings for this iterator
+ * @throws AccumuloSecurityException
+ * thrown if the user does not have the ability to set properties on the table
- * @throws AccumuloException
+ * @throws TableNotFoundException
+ * throw if the table no longer exists
+ */
+ public IteratorSetting getIteratorSetting(String tableName, String name, IteratorScope scope) throws AccumuloSecurityException, AccumuloException,
+ TableNotFoundException;
+
+ /**
+ * Get a list of iterators for this table.
+ *
+ * @param tableName
+ * the name of the table
+ * @return a set of iterator names
- * @throws AccumuloSecurityException
- * @throws AccumuloException
- * @throws TableNotFoundException
+ */
+ public Map<String,EnumSet<IteratorScope>> listIterators(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
+
+ /**
+ * Check whether a given iterator configuration conflicts with existing configuration; in particular, determine if the name or priority are already in use for
+ * the specified scopes.
+ *
+ * @param tableName
+ * the name of the table
+ * @param setting
+ * object specifying the properties of the iterator
- * @throws AccumuloException
- * @throws TableNotFoundException
+ * @throws IllegalStateException
+ * if the setting conflicts with any existing iterators
+ */
+ public void checkIteratorConflicts(String tableName, IteratorSetting setting, EnumSet<IteratorScope> scopes) throws AccumuloException, TableNotFoundException;
+
+ /**
+ * Add a new constraint to a table.
+ *
+ * @param tableName
+ * the name of the table
+ * @param constraintClassName
+ * the full name of the constraint class
+ * @return the unique number assigned to the constraint
+ * @throws AccumuloException
+ * thrown if the constraint has already been added to the table or if there are errors in the configuration of existing constraints
+ * @throws AccumuloSecurityException
+ * thrown if the user doesn't have permission to add the constraint
- * @throws TableNotFoundException
+ * @since 1.5.0
+ */
+ public int addConstraint(String tableName, String constraintClassName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
+ /**
+ * Remove a constraint from a table.
+ *
+ * @param tableName
+ * the name of the table
+ * @param number
+ * the unique number assigned to the constraint
- * @throws AccumuloException
+ * @throws AccumuloSecurityException
+ * thrown if the user doesn't have permission to remove the constraint
+ * @since 1.5.0
+ */
+ public void removeConstraint(String tableName, int number) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * List constraints on a table with their assigned numbers.
+ *
+ * @param tableName
+ * the name of the table
+ * @return a map from constraint class name to assigned number
+ * @throws AccumuloException
+ * thrown if there are errors in the configuration of existing constraints
- * @throws TableNotFoundException
+ * @since 1.5.0
+ */
+ public Map<String,Integer> listConstraints(String tableName) throws AccumuloException, TableNotFoundException;
+
+ /**
+ * Test to see if the instance can load the given class as the given type. This check uses the table classpath if it is set.
+ *
- * @param className
- * @param asTypeName
+ * @return true if the instance can load the given class as the given type, false otherwise
- * @throws AccumuloException
- *
+ *
+ * @since 1.5.0
+ */
+ public boolean testClassLoad(String tableName, final String className, final String asTypeName) throws AccumuloException, AccumuloSecurityException,
+ TableNotFoundException;
+}
[14/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftTransportPool.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/ThriftTransportPool.java
index f123289,0000000..d2113fb
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftTransportPool.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftTransportPool.java
@@@ -1,671 -1,0 +1,683 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.impl;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.security.SecurityPermission;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.TTimeoutTransport;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.log4j.Logger;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+
+public class ThriftTransportPool {
+ private static SecurityPermission TRANSPORT_POOL_PERMISSION = new SecurityPermission("transportPoolPermission");
+
+ private static final Random random = new Random();
+ private long killTime = 1000 * 3;
+
+ private Map<ThriftTransportKey,List<CachedConnection>> cache = new HashMap<ThriftTransportKey,List<CachedConnection>>();
+ private Map<ThriftTransportKey,Long> errorCount = new HashMap<ThriftTransportKey,Long>();
+ private Map<ThriftTransportKey,Long> errorTime = new HashMap<ThriftTransportKey,Long>();
+ private Set<ThriftTransportKey> serversWarnedAbout = new HashSet<ThriftTransportKey>();
+
+ private CountDownLatch closerExitLatch;
+
+ private static final Logger log = Logger.getLogger(ThriftTransportPool.class);
+
+ private static final Long ERROR_THRESHOLD = 20l;
+ private static final int STUCK_THRESHOLD = 2 * 60 * 1000;
+
+ private static class CachedConnection {
+
+ public CachedConnection(CachedTTransport t) {
+ this.transport = t;
+ }
+
+ void setReserved(boolean reserved) {
+ this.transport.setReserved(reserved);
+ }
+
+ boolean isReserved() {
+ return this.transport.reserved;
+ }
+
+ CachedTTransport transport;
+
+ long lastReturnTime;
+ }
+
+ public static class TransportPoolShutdownException extends RuntimeException {
+ private static final long serialVersionUID = 1L;
+ }
+
+ private static class Closer implements Runnable {
+ final ThriftTransportPool pool;
+ private CountDownLatch closerExitLatch;
+
+ public Closer(ThriftTransportPool pool, CountDownLatch closerExitLatch) {
+ this.pool = pool;
+ this.closerExitLatch = closerExitLatch;
+ }
+
+ private void closeConnections() {
+ while (true) {
+
+ ArrayList<CachedConnection> connectionsToClose = new ArrayList<CachedConnection>();
+
+ synchronized (pool) {
+ for (List<CachedConnection> ccl : pool.getCache().values()) {
+ Iterator<CachedConnection> iter = ccl.iterator();
+ while (iter.hasNext()) {
+ CachedConnection cachedConnection = iter.next();
+
+ if (!cachedConnection.isReserved() && System.currentTimeMillis() - cachedConnection.lastReturnTime > pool.killTime) {
+ connectionsToClose.add(cachedConnection);
+ iter.remove();
+ }
+ }
+ }
+
+ for (List<CachedConnection> ccl : pool.getCache().values()) {
+ for (CachedConnection cachedConnection : ccl) {
+ cachedConnection.transport.checkForStuckIO(STUCK_THRESHOLD);
+ }
+ }
+
+ Iterator<Entry<ThriftTransportKey,Long>> iter = pool.errorTime.entrySet().iterator();
+ while (iter.hasNext()) {
+ Entry<ThriftTransportKey,Long> entry = iter.next();
+ long delta = System.currentTimeMillis() - entry.getValue();
+ if (delta >= STUCK_THRESHOLD) {
+ pool.errorCount.remove(entry.getKey());
+ iter.remove();
+ }
+ }
+ }
+
+ // close connections outside of sync block
+ for (CachedConnection cachedConnection : connectionsToClose) {
+ cachedConnection.transport.close();
+ }
+
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
++ @Override
+ public void run() {
+ try {
+ closeConnections();
+ } catch (TransportPoolShutdownException e) {
+ } finally {
+ closerExitLatch.countDown();
+ }
+ }
+ }
+
+ static class CachedTTransport extends TTransport {
+
+ private ThriftTransportKey cacheKey;
+ private TTransport wrappedTransport;
+ private boolean sawError = false;
+
+ private volatile String ioThreadName = null;
+ private volatile long ioStartTime = 0;
+ private volatile boolean reserved = false;
+
+ private String stuckThreadName = null;
+
+ int ioCount = 0;
+ int lastIoCount = -1;
+
+ private void sawError(Exception e) {
+ sawError = true;
+ }
+
+ final void setReserved(boolean reserved) {
+ this.reserved = reserved;
+ if (reserved) {
+ ioThreadName = Thread.currentThread().getName();
+ ioCount = 0;
+ lastIoCount = -1;
+ } else {
+ if ((ioCount & 1) == 1) {
+ // connection unreserved, but it seems io may still be
+ // happening
+ log.warn("Connection returned to thrift connection pool that may still be in use " + ioThreadName + " " + Thread.currentThread().getName(),
+ new Exception());
+ }
+
+ ioCount = 0;
+ lastIoCount = -1;
+ ioThreadName = null;
+ }
+ checkForStuckIO(STUCK_THRESHOLD);
+ }
+
+ final void checkForStuckIO(long threshold) {
+ /*
+ * checking for stuck io needs to be light weight.
+ *
+ * Tried to call System.currentTimeMillis() and Thread.currentThread() before every io operation.... this dramatically slowed things down. So switched to
+ * incrementing a counter before and after each io operation.
+ */
+
+ if ((ioCount & 1) == 1) {
+ // when ioCount is odd, it means I/O is currently happening
+ if (ioCount == lastIoCount) {
+ // still doing same I/O operation as last time this
+ // functions was called
+ long delta = System.currentTimeMillis() - ioStartTime;
+ if (delta >= threshold && stuckThreadName == null) {
+ stuckThreadName = ioThreadName;
+ log.warn("Thread \"" + ioThreadName + "\" stuck on IO to " + cacheKey + " for at least " + delta + " ms");
+ }
+ } else {
+ // remember this ioCount and the time we saw it, need to see
+ // if it changes
+ lastIoCount = ioCount;
+ ioStartTime = System.currentTimeMillis();
+
+ if (stuckThreadName != null) {
+ // doing I/O, but ioCount changed so no longer stuck
+ log.info("Thread \"" + stuckThreadName + "\" no longer stuck on IO to " + cacheKey + " sawError = " + sawError);
+ stuckThreadName = null;
+ }
+ }
+ } else {
+ // I/O is not currently happening
+ if (stuckThreadName != null) {
+ // no longer stuck, and was stuck in the past
+ log.info("Thread \"" + stuckThreadName + "\" no longer stuck on IO to " + cacheKey + " sawError = " + sawError);
+ stuckThreadName = null;
+ }
+ }
+ }
+
+ public CachedTTransport(TTransport transport, ThriftTransportKey cacheKey2) {
+ this.wrappedTransport = transport;
+ this.cacheKey = cacheKey2;
+ }
+
++ @Override
+ public boolean isOpen() {
+ return wrappedTransport.isOpen();
+ }
+
++ @Override
+ public void open() throws TTransportException {
+ try {
+ ioCount++;
+ wrappedTransport.open();
+ } catch (TTransportException tte) {
+ sawError(tte);
+ throw tte;
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public int read(byte[] arg0, int arg1, int arg2) throws TTransportException {
+ try {
+ ioCount++;
+ return wrappedTransport.read(arg0, arg1, arg2);
+ } catch (TTransportException tte) {
+ sawError(tte);
+ throw tte;
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public int readAll(byte[] arg0, int arg1, int arg2) throws TTransportException {
+ try {
+ ioCount++;
+ return wrappedTransport.readAll(arg0, arg1, arg2);
+ } catch (TTransportException tte) {
+ sawError(tte);
+ throw tte;
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public void write(byte[] arg0, int arg1, int arg2) throws TTransportException {
+ try {
+ ioCount++;
+ wrappedTransport.write(arg0, arg1, arg2);
+ } catch (TTransportException tte) {
+ sawError(tte);
+ throw tte;
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public void write(byte[] arg0) throws TTransportException {
+ try {
+ ioCount++;
+ wrappedTransport.write(arg0);
+ } catch (TTransportException tte) {
+ sawError(tte);
+ throw tte;
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public void close() {
+ try {
+ ioCount++;
+ wrappedTransport.close();
+ } finally {
+ ioCount++;
+ }
+
+ }
+
++ @Override
+ public void flush() throws TTransportException {
+ try {
+ ioCount++;
+ wrappedTransport.flush();
+ } catch (TTransportException tte) {
+ sawError(tte);
+ throw tte;
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public boolean peek() {
+ try {
+ ioCount++;
+ return wrappedTransport.peek();
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public byte[] getBuffer() {
+ try {
+ ioCount++;
+ return wrappedTransport.getBuffer();
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public int getBufferPosition() {
+ try {
+ ioCount++;
+ return wrappedTransport.getBufferPosition();
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public int getBytesRemainingInBuffer() {
+ try {
+ ioCount++;
+ return wrappedTransport.getBytesRemainingInBuffer();
+ } finally {
+ ioCount++;
+ }
+ }
+
++ @Override
+ public void consumeBuffer(int len) {
+ try {
+ ioCount++;
+ wrappedTransport.consumeBuffer(len);
+ } finally {
+ ioCount++;
+ }
+ }
+
+ public ThriftTransportKey getCacheKey() {
+ return cacheKey;
+ }
+
+ }
+
+ private ThriftTransportPool() {}
+
+ public TTransport getTransport(String location, int port) throws TTransportException {
+ return getTransport(location, port, 0);
+ }
+
+ public TTransport getTransportWithDefaultTimeout(InetSocketAddress addr, AccumuloConfiguration conf) throws TTransportException {
+ return getTransport(addr.getAddress().getHostAddress(), addr.getPort(), conf.getTimeInMillis(Property.GENERAL_RPC_TIMEOUT));
+ }
+
+ public TTransport getTransport(InetSocketAddress addr, long timeout) throws TTransportException {
+ return getTransport(addr.getAddress().getHostAddress(), addr.getPort(), timeout);
+ }
+
+ public TTransport getTransportWithDefaultTimeout(String location, int port, AccumuloConfiguration conf) throws TTransportException {
+ return getTransport(location, port, conf.getTimeInMillis(Property.GENERAL_RPC_TIMEOUT));
+ }
+
+ Pair<String,TTransport> getAnyTransport(List<ThriftTransportKey> servers, boolean preferCachedConnection) throws TTransportException {
+
+ servers = new ArrayList<ThriftTransportKey>(servers);
+
+ if (preferCachedConnection) {
+ HashSet<ThriftTransportKey> serversSet = new HashSet<ThriftTransportKey>(servers);
+
+ synchronized (this) {
+
+ // randomly pick a server from the connection cache
+ serversSet.retainAll(getCache().keySet());
+
+ if (serversSet.size() > 0) {
+ ArrayList<ThriftTransportKey> cachedServers = new ArrayList<ThriftTransportKey>(serversSet);
+ Collections.shuffle(cachedServers, random);
+
+ for (ThriftTransportKey ttk : cachedServers) {
+ for (CachedConnection cachedConnection : getCache().get(ttk)) {
+ if (!cachedConnection.isReserved()) {
+ cachedConnection.setReserved(true);
+ if (log.isTraceEnabled())
+ log.trace("Using existing connection to " + ttk.getLocation() + ":" + ttk.getPort());
+ return new Pair<String,TTransport>(ttk.getLocation() + ":" + ttk.getPort(), cachedConnection.transport);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ int retryCount = 0;
+ while (servers.size() > 0 && retryCount < 10) {
+ int index = random.nextInt(servers.size());
+ ThriftTransportKey ttk = servers.get(index);
+
+ if (!preferCachedConnection) {
+ synchronized (this) {
+ List<CachedConnection> cachedConnList = getCache().get(ttk);
+ if (cachedConnList != null) {
+ for (CachedConnection cachedConnection : cachedConnList) {
+ if (!cachedConnection.isReserved()) {
+ cachedConnection.setReserved(true);
+ if (log.isTraceEnabled())
+ log.trace("Using existing connection to " + ttk.getLocation() + ":" + ttk.getPort() + " timeout " + ttk.getTimeout());
+ return new Pair<String,TTransport>(ttk.getLocation() + ":" + ttk.getPort(), cachedConnection.transport);
+ }
+ }
+ }
+ }
+ }
+
+ try {
+ return new Pair<String,TTransport>(ttk.getLocation() + ":" + ttk.getPort(), createNewTransport(ttk));
+ } catch (TTransportException tte) {
+ log.debug("Failed to connect to " + servers.get(index), tte);
+ servers.remove(index);
+ retryCount++;
+ }
+ }
+
+ throw new TTransportException("Failed to connect to a server");
+ }
+
+ public TTransport getTransport(String location, int port, long milliseconds) throws TTransportException {
+ return getTransport(new ThriftTransportKey(location, port, milliseconds));
+ }
+
+ private TTransport getTransport(ThriftTransportKey cacheKey) throws TTransportException {
+ synchronized (this) {
+ // atomically reserve location if it exist in cache
+ List<CachedConnection> ccl = getCache().get(cacheKey);
+
+ if (ccl == null) {
+ ccl = new LinkedList<CachedConnection>();
+ getCache().put(cacheKey, ccl);
+ }
+
+ for (CachedConnection cachedConnection : ccl) {
+ if (!cachedConnection.isReserved()) {
+ cachedConnection.setReserved(true);
+ if (log.isTraceEnabled())
+ log.trace("Using existing connection to " + cacheKey.getLocation() + ":" + cacheKey.getPort());
+ return cachedConnection.transport;
+ }
+ }
+ }
+
+ return createNewTransport(cacheKey);
+ }
+
+ private TTransport createNewTransport(ThriftTransportKey cacheKey) throws TTransportException {
+ TTransport transport;
+ if (cacheKey.getTimeout() == 0) {
+ transport = AddressUtil.createTSocket(cacheKey.getLocation(), cacheKey.getPort());
+ } else {
+ try {
+ transport = TTimeoutTransport.create(AddressUtil.parseAddress(cacheKey.getLocation(), cacheKey.getPort()), cacheKey.getTimeout());
+ } catch (IOException ex) {
+ throw new TTransportException(ex);
+ }
+ }
+ transport = ThriftUtil.transportFactory().getTransport(transport);
+ transport.open();
+
+ if (log.isTraceEnabled())
+ log.trace("Creating new connection to connection to " + cacheKey.getLocation() + ":" + cacheKey.getPort());
+
+ CachedTTransport tsc = new CachedTTransport(transport, cacheKey);
+
+ CachedConnection cc = new CachedConnection(tsc);
+ cc.setReserved(true);
+
+ try {
+ synchronized (this) {
+ List<CachedConnection> ccl = getCache().get(cacheKey);
+
+ if (ccl == null) {
+ ccl = new LinkedList<CachedConnection>();
+ getCache().put(cacheKey, ccl);
+ }
+
+ ccl.add(cc);
+ }
+ } catch (TransportPoolShutdownException e) {
+ cc.transport.close();
+ throw e;
+ }
+ return cc.transport;
+ }
+
+ public void returnTransport(TTransport tsc) {
+ if (tsc == null) {
+ return;
+ }
+
+ boolean existInCache = false;
+ CachedTTransport ctsc = (CachedTTransport) tsc;
+
+ ArrayList<CachedConnection> closeList = new ArrayList<ThriftTransportPool.CachedConnection>();
+
+ synchronized (this) {
+ List<CachedConnection> ccl = getCache().get(ctsc.getCacheKey());
+ for (Iterator<CachedConnection> iterator = ccl.iterator(); iterator.hasNext();) {
+ CachedConnection cachedConnection = iterator.next();
+ if (cachedConnection.transport == tsc) {
+ if (ctsc.sawError) {
+ closeList.add(cachedConnection);
+ iterator.remove();
+
+ if (log.isTraceEnabled())
+ log.trace("Returned connection had error " + ctsc.getCacheKey());
+
+ Long ecount = errorCount.get(ctsc.getCacheKey());
+ if (ecount == null)
+ ecount = 0l;
+ ecount++;
+ errorCount.put(ctsc.getCacheKey(), ecount);
+
+ Long etime = errorTime.get(ctsc.getCacheKey());
+ if (etime == null) {
+ errorTime.put(ctsc.getCacheKey(), System.currentTimeMillis());
+ }
+
+ if (ecount >= ERROR_THRESHOLD && !serversWarnedAbout.contains(ctsc.getCacheKey())) {
+ log.warn("Server " + ctsc.getCacheKey() + " had " + ecount + " failures in a short time period, will not complain anymore ");
+ serversWarnedAbout.add(ctsc.getCacheKey());
+ }
+
+ cachedConnection.setReserved(false);
+
+ } else {
+
+ if (log.isTraceEnabled())
+ log.trace("Returned connection " + ctsc.getCacheKey() + " ioCount : " + cachedConnection.transport.ioCount);
+
+ cachedConnection.lastReturnTime = System.currentTimeMillis();
+ cachedConnection.setReserved(false);
+ }
+ existInCache = true;
+ break;
+ }
+ }
+
+ // remove all unreserved cached connection when a sever has an error, not just the connection that was returned
+ if (ctsc.sawError) {
+ for (Iterator<CachedConnection> iterator = ccl.iterator(); iterator.hasNext();) {
+ CachedConnection cachedConnection = iterator.next();
+ if (!cachedConnection.isReserved()) {
+ closeList.add(cachedConnection);
+ iterator.remove();
+ }
+ }
+ }
+ }
+
+ // close outside of sync block
+ for (CachedConnection cachedConnection : closeList) {
+ try {
+ cachedConnection.transport.close();
+ } catch (Exception e) {
+ log.debug("Failed to close connection w/ errors", e);
+ }
+ }
+
+ if (!existInCache) {
+ log.warn("Returned tablet server connection to cache that did not come from cache");
+ // close outside of sync block
+ tsc.close();
+ }
+ }
+
+ /**
+ * Set the time after which idle connections should be closed
- *
- * @param time
+ */
+ public synchronized void setIdleTime(long time) {
+ this.killTime = time;
+ log.debug("Set thrift transport pool idle time to " + time);
+ }
+
+ private static ThriftTransportPool instance = new ThriftTransportPool();
+ private static final AtomicBoolean daemonStarted = new AtomicBoolean(false);
+
+ public static ThriftTransportPool getInstance() {
+ SecurityManager sm = System.getSecurityManager();
+ if (sm != null) {
+ sm.checkPermission(TRANSPORT_POOL_PERMISSION);
+ }
+
+ if (daemonStarted.compareAndSet(false, true)) {
+ CountDownLatch closerExitLatch = new CountDownLatch(1);
+ new Daemon(new Closer(instance, closerExitLatch), "Thrift Connection Pool Checker").start();
+ instance.setCloserExitLatch(closerExitLatch);
+ }
+ return instance;
+ }
+
+ private synchronized void setCloserExitLatch(CountDownLatch closerExitLatch) {
+ this.closerExitLatch = closerExitLatch;
+ }
+
+ public void shutdown() {
+ synchronized (this) {
+ if (cache == null)
+ return;
+
+ // close any connections in the pool... even ones that are in use
+ for (List<CachedConnection> ccl : getCache().values()) {
+ Iterator<CachedConnection> iter = ccl.iterator();
+ while (iter.hasNext()) {
+ CachedConnection cc = iter.next();
+ try {
+ cc.transport.close();
+ } catch (Exception e) {
+ log.debug("Error closing transport during shutdown", e);
+ }
+ }
+ }
+
+ // this will render the pool unusable and cause the background thread to exit
+ this.cache = null;
+ }
+
+ try {
+ closerExitLatch.await();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private Map<ThriftTransportKey,List<CachedConnection>> getCache() {
+ if (cache == null)
+ throw new TransportPoolShutdownException();
+ return cache;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
index ee4aca5,0000000..d7be37c
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
@@@ -1,511 -1,0 +1,510 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapred;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.mapreduce.lib.util.OutputConfigurator;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.SecurityErrorCode;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.OutputFormat;
+import org.apache.hadoop.mapred.RecordWriter;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.util.Progressable;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * This class allows MapReduce jobs to use Accumulo as the sink for data. This {@link OutputFormat} accepts keys and values of type {@link Text} (for a table
+ * name) and {@link Mutation} from the Map and Reduce functions.
+ *
+ * The user must specify the following via static configurator methods:
+ *
+ * <ul>
+ * <li>{@link AccumuloOutputFormat#setConnectorInfo(JobConf, String, AuthenticationToken)}
+ * <li>{@link AccumuloOutputFormat#setZooKeeperInstance(JobConf, String, String)} OR {@link AccumuloOutputFormat#setMockInstance(JobConf, String)}
+ * </ul>
+ *
+ * Other static methods are optional.
+ */
+public class AccumuloOutputFormat implements OutputFormat<Text,Mutation> {
+
+ private static final Class<?> CLASS = AccumuloOutputFormat.class;
+ protected static final Logger log = Logger.getLogger(CLASS);
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
+ * conversion to a string, and is not intended to be secure.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission if {@link #setCreateTables(JobConf, boolean)} is set to true)
+ * @param token
+ * the user's password
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public static void setConnectorInfo(JobConf job, String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ OutputConfigurator.setConnectorInfo(CLASS, job, principal, token);
+ }
+
+ /**
+ * Determines if the connector has been configured.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the connector has been configured, false otherwise
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static Boolean isConnectorInfoSet(JobConf job) {
+ return OutputConfigurator.isConnectorInfoSet(CLASS, job);
+ }
+
+ /**
+ * Gets the principal from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static String getPrincipal(JobConf job) {
+ return OutputConfigurator.getPrincipal(CLASS, job);
+ }
+
+ /**
+ * Gets the serialized token class from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static String getTokenClass(JobConf job) {
+ return OutputConfigurator.getTokenClass(CLASS, job);
+ }
+
+ /**
+ * Gets the password from the configuration. WARNING: The password is stored in the Configuration and shared with all MapReduce tasks; It is BASE64 encoded to
+ * provide a charset safe conversion to a string, and is not intended to be secure.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the decoded user password
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static byte[] getToken(JobConf job) {
+ return OutputConfigurator.getToken(CLASS, job);
+ }
+
+ /**
+ * Configures a {@link ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @param zooKeepers
+ * a comma-separated list of zookeeper servers
+ * @since 1.5.0
+ */
+ public static void setZooKeeperInstance(JobConf job, String instanceName, String zooKeepers) {
+ OutputConfigurator.setZooKeeperInstance(CLASS, job, instanceName, zooKeepers);
+ }
+
+ /**
+ * Configures a {@link MockInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @since 1.5.0
+ */
+ public static void setMockInstance(JobConf job, String instanceName) {
+ OutputConfigurator.setMockInstance(CLASS, job, instanceName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link Instance} based on the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return an Accumulo instance
+ * @since 1.5.0
+ * @see #setZooKeeperInstance(JobConf, String, String)
+ * @see #setMockInstance(JobConf, String)
+ */
+ protected static Instance getInstance(JobConf job) {
+ return OutputConfigurator.getInstance(CLASS, job);
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param level
+ * the logging level
+ * @since 1.5.0
+ */
+ public static void setLogLevel(JobConf job, Level level) {
+ OutputConfigurator.setLogLevel(CLASS, job, level);
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the log level
+ * @since 1.5.0
+ * @see #setLogLevel(JobConf, Level)
+ */
+ protected static Level getLogLevel(JobConf job) {
+ return OutputConfigurator.getLogLevel(CLASS, job);
+ }
+
+ /**
+ * Sets the default table name to use if one emits a null in place of a table name for a given mutation. Table names can only be alpha-numeric and
+ * underscores.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.5.0
+ */
+ public static void setDefaultTableName(JobConf job, String tableName) {
+ OutputConfigurator.setDefaultTableName(CLASS, job, tableName);
+ }
+
+ /**
+ * Gets the default table name from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the default table name
+ * @since 1.5.0
+ * @see #setDefaultTableName(JobConf, String)
+ */
+ protected static String getDefaultTableName(JobConf job) {
+ return OutputConfigurator.getDefaultTableName(CLASS, job);
+ }
+
+ /**
+ * Sets the configuration for for the job's {@link BatchWriter} instances. If not set, a new {@link BatchWriterConfig}, with sensible built-in defaults is
+ * used. Setting the configuration multiple times overwrites any previous configuration.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param bwConfig
+ * the configuration for the {@link BatchWriter}
+ * @since 1.5.0
+ */
+ public static void setBatchWriterOptions(JobConf job, BatchWriterConfig bwConfig) {
+ OutputConfigurator.setBatchWriterOptions(CLASS, job, bwConfig);
+ }
+
+ /**
+ * Gets the {@link BatchWriterConfig} settings.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the configuration object
+ * @since 1.5.0
+ * @see #setBatchWriterOptions(JobConf, BatchWriterConfig)
+ */
+ protected static BatchWriterConfig getBatchWriterOptions(JobConf job) {
+ return OutputConfigurator.getBatchWriterOptions(CLASS, job);
+ }
+
+ /**
+ * Sets the directive to create new tables, as necessary. Table names can only be alpha-numeric and underscores.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setCreateTables(JobConf job, boolean enableFeature) {
+ OutputConfigurator.setCreateTables(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether tables are permitted to be created as needed.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the feature is disabled, false otherwise
+ * @since 1.5.0
+ * @see #setCreateTables(JobConf, boolean)
+ */
+ protected static Boolean canCreateTables(JobConf job) {
+ return OutputConfigurator.canCreateTables(CLASS, job);
+ }
+
+ /**
+ * Sets the directive to use simulation mode for this job. In simulation mode, no output is produced. This is useful for testing.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setSimulationMode(JobConf job, boolean enableFeature) {
+ OutputConfigurator.setSimulationMode(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether this feature is enabled.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setSimulationMode(JobConf, boolean)
+ */
+ protected static Boolean getSimulationMode(JobConf job) {
+ return OutputConfigurator.getSimulationMode(CLASS, job);
+ }
+
+ /**
+ * A base class to be used to create {@link RecordWriter} instances that write to Accumulo.
+ */
+ protected static class AccumuloRecordWriter implements RecordWriter<Text,Mutation> {
+ private MultiTableBatchWriter mtbw = null;
+ private HashMap<Text,BatchWriter> bws = null;
+ private Text defaultTableName = null;
+
+ private boolean simulate = false;
+ private boolean createTables = false;
+
+ private long mutCount = 0;
+ private long valCount = 0;
+
+ private Connector conn;
+
+ protected AccumuloRecordWriter(JobConf job) throws AccumuloException, AccumuloSecurityException, IOException {
+ Level l = getLogLevel(job);
+ if (l != null)
+ log.setLevel(getLogLevel(job));
+ this.simulate = getSimulationMode(job);
+ this.createTables = canCreateTables(job);
+
+ if (simulate)
+ log.info("Simulating output only. No writes to tables will occur");
+
+ this.bws = new HashMap<Text,BatchWriter>();
+
+ String tname = getDefaultTableName(job);
+ this.defaultTableName = (tname == null) ? null : new Text(tname);
+
+ if (!simulate) {
+ this.conn = getInstance(job).getConnector(getPrincipal(job), CredentialHelper.extractToken(getTokenClass(job), getToken(job)));
+ mtbw = conn.createMultiTableBatchWriter(getBatchWriterOptions(job));
+ }
+ }
+
+ /**
+ * Push a mutation into a table. If table is null, the defaultTable will be used. If canCreateTable is set, the table will be created if it does not exist.
+ * The table name must only contain alphanumerics and underscore.
+ */
+ @Override
+ public void write(Text table, Mutation mutation) throws IOException {
+ if (table == null || table.toString().isEmpty())
+ table = this.defaultTableName;
+
+ if (!simulate && table == null)
+ throw new IOException("No table or default table specified. Try simulation mode next time");
+
+ ++mutCount;
+ valCount += mutation.size();
+ printMutation(table, mutation);
+
+ if (simulate)
+ return;
+
+ if (!bws.containsKey(table))
+ try {
+ addTable(table);
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new IOException(e);
+ }
+
+ try {
+ bws.get(table).addMutation(mutation);
+ } catch (MutationsRejectedException e) {
+ throw new IOException(e);
+ }
+ }
+
+ public void addTable(Text tableName) throws AccumuloException, AccumuloSecurityException {
+ if (simulate) {
+ log.info("Simulating adding table: " + tableName);
+ return;
+ }
+
+ log.debug("Adding table: " + tableName);
+ BatchWriter bw = null;
+ String table = tableName.toString();
+
+ if (createTables && !conn.tableOperations().exists(table)) {
+ try {
+ conn.tableOperations().create(table);
+ } catch (AccumuloSecurityException e) {
+ log.error("Accumulo security violation creating " + table, e);
+ throw e;
+ } catch (TableExistsException e) {
+ // Shouldn't happen
+ }
+ }
+
+ try {
+ bw = mtbw.getBatchWriter(table);
+ } catch (TableNotFoundException e) {
+ log.error("Accumulo table " + table + " doesn't exist and cannot be created.", e);
+ throw new AccumuloException(e);
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (AccumuloSecurityException e) {
+ throw e;
+ }
+
+ if (bw != null)
+ bws.put(tableName, bw);
+ }
+
+ private int printMutation(Text table, Mutation m) {
+ if (log.isTraceEnabled()) {
+ log.trace(String.format("Table %s row key: %s", table, hexDump(m.getRow())));
+ for (ColumnUpdate cu : m.getUpdates()) {
+ log.trace(String.format("Table %s column: %s:%s", table, hexDump(cu.getColumnFamily()), hexDump(cu.getColumnQualifier())));
+ log.trace(String.format("Table %s security: %s", table, new ColumnVisibility(cu.getColumnVisibility()).toString()));
+ log.trace(String.format("Table %s value: %s", table, hexDump(cu.getValue())));
+ }
+ }
+ return m.getUpdates().size();
+ }
+
+ private String hexDump(byte[] ba) {
+ StringBuilder sb = new StringBuilder();
+ for (byte b : ba) {
+ if ((b > 0x20) && (b < 0x7e))
+ sb.append((char) b);
+ else
+ sb.append(String.format("x%02x", b));
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public void close(Reporter reporter) throws IOException {
+ log.debug("mutations written: " + mutCount + ", values written: " + valCount);
+ if (simulate)
+ return;
+
+ try {
+ mtbw.close();
+ } catch (MutationsRejectedException e) {
+ if (e.getAuthorizationFailuresMap().size() >= 0) {
+ HashMap<String,Set<SecurityErrorCode>> tables = new HashMap<String,Set<SecurityErrorCode>>();
+ for (Entry<KeyExtent,Set<SecurityErrorCode>> ke : e.getAuthorizationFailuresMap().entrySet()) {
+ Set<SecurityErrorCode> secCodes = tables.get(ke.getKey().getTableId().toString());
+ if (secCodes == null) {
+ secCodes = new HashSet<SecurityErrorCode>();
+ tables.put(ke.getKey().getTableId().toString(), secCodes);
+ }
+ secCodes.addAll(ke.getValue());
+ }
+
+ log.error("Not authorized to write to tables : " + tables);
+ }
+
+ if (e.getConstraintViolationSummaries().size() > 0) {
+ log.error("Constraint violations : " + e.getConstraintViolationSummaries().size());
+ }
+ }
+ }
+ }
+
+ @Override
+ public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException {
+ if (!isConnectorInfoSet(job))
+ throw new IOException("Connector info has not been set.");
+ try {
+ // if the instance isn't configured, it will complain here
+ Connector c = getInstance(job).getConnector(getPrincipal(job), CredentialHelper.extractToken(getTokenClass(job), getToken(job)));
+ if (!c.securityOperations().authenticateUser(getPrincipal(job), CredentialHelper.extractToken(getTokenClass(job), getToken(job))))
+ throw new IOException("Unable to authenticate user");
+ } catch (AccumuloException e) {
+ throw new IOException(e);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public RecordWriter<Text,Mutation> getRecordWriter(FileSystem ignored, JobConf job, String name, Progressable progress) throws IOException {
+ try {
+ return new AccumuloRecordWriter(job);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
index 16efa89,0000000..bc568e8
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
@@@ -1,925 -1,0 +1,924 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapred;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientSideIteratorScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.RowIterator;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableDeletedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.impl.OfflineScanner;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.TabletLocator;
+import org.apache.accumulo.core.client.mapreduce.lib.util.InputConfigurator;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * This abstract {@link InputFormat} class allows MapReduce jobs to use Accumulo as the source of K,V pairs.
+ * <p>
+ * Subclasses must implement a {@link #getRecordReader(InputSplit, JobConf, Reporter)} to provide a {@link RecordReader} for K,V.
+ * <p>
+ * A static base class, RecordReaderBase, is provided to retrieve Accumulo {@link Key}/{@link Value} pairs, but one must implement its
+ * {@link RecordReaderBase#next(Object, Object)} to transform them to the desired generic types K,V.
+ * <p>
+ * See {@link AccumuloInputFormat} for an example implementation.
+ */
+public abstract class InputFormatBase<K,V> implements InputFormat<K,V> {
+
+ private static final Class<?> CLASS = AccumuloInputFormat.class;
+ protected static final Logger log = Logger.getLogger(CLASS);
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
+ * conversion to a string, and is not intended to be secure.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission)
+ * @param token
+ * the user's password
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public static void setConnectorInfo(JobConf job, String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ InputConfigurator.setConnectorInfo(CLASS, job, principal, token);
+ }
+
+ /**
+ * Determines if the connector has been configured.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the connector has been configured, false otherwise
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static Boolean isConnectorInfoSet(JobConf job) {
+ return InputConfigurator.isConnectorInfoSet(CLASS, job);
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static String getPrincipal(JobConf job) {
+ return InputConfigurator.getPrincipal(CLASS, job);
+ }
+
+ /**
+ * Gets the serialized token class from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static String getTokenClass(JobConf job) {
+ return InputConfigurator.getTokenClass(CLASS, job);
+ }
+
+ /**
+ * Gets the password from the configuration. WARNING: The password is stored in the Configuration and shared with all MapReduce tasks; It is BASE64 encoded to
+ * provide a charset safe conversion to a string, and is not intended to be secure.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the decoded user password
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static byte[] getToken(JobConf job) {
+ return InputConfigurator.getToken(CLASS, job);
+ }
+
+ /**
+ * Configures a {@link ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @param zooKeepers
+ * a comma-separated list of zookeeper servers
+ * @since 1.5.0
+ */
+ public static void setZooKeeperInstance(JobConf job, String instanceName, String zooKeepers) {
+ InputConfigurator.setZooKeeperInstance(CLASS, job, instanceName, zooKeepers);
+ }
+
+ /**
+ * Configures a {@link MockInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param instanceName
+ * the Accumulo instance name
+ * @since 1.5.0
+ */
+ public static void setMockInstance(JobConf job, String instanceName) {
+ InputConfigurator.setMockInstance(CLASS, job, instanceName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link Instance} based on the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return an Accumulo instance
+ * @since 1.5.0
+ * @see #setZooKeeperInstance(JobConf, String, String)
+ * @see #setMockInstance(JobConf, String)
+ */
+ protected static Instance getInstance(JobConf job) {
+ return InputConfigurator.getInstance(CLASS, job);
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param level
+ * the logging level
+ * @since 1.5.0
+ */
+ public static void setLogLevel(JobConf job, Level level) {
+ InputConfigurator.setLogLevel(CLASS, job, level);
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the log level
+ * @since 1.5.0
+ * @see #setLogLevel(JobConf, Level)
+ */
+ protected static Level getLogLevel(JobConf job) {
+ return InputConfigurator.getLogLevel(CLASS, job);
+ }
+
+ /**
+ * Sets the name of the input table, over which this job will scan.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.5.0
+ */
+ public static void setInputTableName(JobConf job, String tableName) {
+ InputConfigurator.setInputTableName(CLASS, job, tableName);
+ }
+
+ /**
+ * Gets the table name from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the table name
+ * @since 1.5.0
+ * @see #setInputTableName(JobConf, String)
+ */
+ protected static String getInputTableName(JobConf job) {
+ return InputConfigurator.getInputTableName(CLASS, job);
+ }
+
+ /**
+ * Sets the {@link Authorizations} used to scan. Must be a subset of the user's authorization. Defaults to the empty set.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param auths
+ * the user's authorizations
+ * @since 1.5.0
+ */
+ public static void setScanAuthorizations(JobConf job, Authorizations auths) {
+ InputConfigurator.setScanAuthorizations(CLASS, job, auths);
+ }
+
+ /**
+ * Gets the authorizations to set for the scans from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the Accumulo scan authorizations
+ * @since 1.5.0
+ * @see #setScanAuthorizations(JobConf, Authorizations)
+ */
+ protected static Authorizations getScanAuthorizations(JobConf job) {
+ return InputConfigurator.getScanAuthorizations(CLASS, job);
+ }
+
+ /**
+ * Sets the input ranges to scan for this job. If not set, the entire table will be scanned.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param ranges
+ * the ranges that will be mapped over
+ * @since 1.5.0
+ */
+ public static void setRanges(JobConf job, Collection<Range> ranges) {
+ InputConfigurator.setRanges(CLASS, job, ranges);
+ }
+
+ /**
+ * Gets the ranges to scan over from a job.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the ranges
+ * @throws IOException
+ * if the ranges have been encoded improperly
+ * @since 1.5.0
+ * @see #setRanges(JobConf, Collection)
+ */
+ protected static List<Range> getRanges(JobConf job) throws IOException {
+ return InputConfigurator.getRanges(CLASS, job);
+ }
+
+ /**
+ * Restricts the columns that will be mapped over for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param columnFamilyColumnQualifierPairs
+ * a pair of {@link Text} objects corresponding to column family and column qualifier. If the column qualifier is null, the entire column family is
+ * selected. An empty set is the default and is equivalent to scanning the all columns.
+ * @since 1.5.0
+ */
+ public static void fetchColumns(JobConf job, Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
+ InputConfigurator.fetchColumns(CLASS, job, columnFamilyColumnQualifierPairs);
+ }
+
+ /**
+ * Gets the columns to be mapped over from this job.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return a set of columns
+ * @since 1.5.0
+ * @see #fetchColumns(JobConf, Collection)
+ */
+ protected static Set<Pair<Text,Text>> getFetchedColumns(JobConf job) {
+ return InputConfigurator.getFetchedColumns(CLASS, job);
+ }
+
+ /**
+ * Encode an iterator on the input for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param cfg
+ * the configuration of the iterator
+ * @since 1.5.0
+ */
+ public static void addIterator(JobConf job, IteratorSetting cfg) {
+ InputConfigurator.addIterator(CLASS, job, cfg);
+ }
+
+ /**
+ * Gets a list of the iterator settings (for iterators to apply to a scanner) from this configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return a list of iterators
+ * @since 1.5.0
+ * @see #addIterator(JobConf, IteratorSetting)
+ */
+ protected static List<IteratorSetting> getIterators(JobConf job) {
+ return InputConfigurator.getIterators(CLASS, job);
+ }
+
+ /**
+ * Controls the automatic adjustment of ranges for this job. This feature merges overlapping ranges, then splits them to align with tablet boundaries.
+ * Disabling this feature will cause exactly one Map task to be created for each specified range. The default setting is enabled. *
+ *
+ * <p>
+ * By default, this feature is <b>enabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @see #setRanges(JobConf, Collection)
+ * @since 1.5.0
+ */
+ public static void setAutoAdjustRanges(JobConf job, boolean enableFeature) {
+ InputConfigurator.setAutoAdjustRanges(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has auto-adjust ranges enabled.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return false if the feature is disabled, true otherwise
+ * @since 1.5.0
+ * @see #setAutoAdjustRanges(JobConf, boolean)
+ */
+ protected static boolean getAutoAdjustRanges(JobConf job) {
+ return InputConfigurator.getAutoAdjustRanges(CLASS, job);
+ }
+
+ /**
+ * Controls the use of the {@link IsolatedScanner} in this job.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setScanIsolation(JobConf job, boolean enableFeature) {
+ InputConfigurator.setScanIsolation(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has isolation enabled.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setScanIsolation(JobConf, boolean)
+ */
+ protected static boolean isIsolated(JobConf job) {
+ return InputConfigurator.isIsolated(CLASS, job);
+ }
+
+ /**
+ * Controls the use of the {@link ClientSideIteratorScanner} in this job. Enabling this feature will cause the iterator stack to be constructed within the Map
+ * task, rather than within the Accumulo TServer. To use this feature, all classes needed for those iterators must be available on the classpath for the task.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setLocalIterators(JobConf job, boolean enableFeature) {
+ InputConfigurator.setLocalIterators(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration uses local iterators.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setLocalIterators(JobConf, boolean)
+ */
+ protected static boolean usesLocalIterators(JobConf job) {
+ return InputConfigurator.usesLocalIterators(CLASS, job);
+ }
+
+ /**
+ * <p>
+ * Enable reading offline tables. By default, this feature is disabled and only online tables are scanned. This will make the map reduce job directly read the
+ * table's files. If the table is not offline, then the job will fail. If the table comes online during the map reduce job, it is likely that the job will
+ * fail.
+ *
+ * <p>
+ * To use this option, the map reduce user will need access to read the Accumulo directory in HDFS.
+ *
+ * <p>
+ * Reading the offline table will create the scan time iterator stack in the map process. So any iterators that are configured for the table will need to be
+ * on the mapper's classpath.
+ *
+ * <p>
+ * One way to use this feature is to clone a table, take the clone offline, and use the clone as the input table for a map reduce job. If you plan to map
+ * reduce over the data many times, it may be better to the compact the table, clone it, take it offline, and use the clone for all map reduce jobs. The
+ * reason to do this is that compaction will reduce each tablet in the table to one file, and it is faster to read from one file.
+ *
+ * <p>
+ * There are two possible advantages to reading a tables file directly out of HDFS. First, you may see better read performance. Second, it will support
+ * speculative execution better. When reading an online table speculative execution can put more load on an already slow tablet server.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setOfflineTableScan(JobConf job, boolean enableFeature) {
+ InputConfigurator.setOfflineTableScan(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has the offline table scan feature enabled.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setOfflineTableScan(JobConf, boolean)
+ */
+ protected static boolean isOfflineScan(JobConf job) {
+ return InputConfigurator.isOfflineScan(CLASS, job);
+ }
+
+ /**
+ * Initializes an Accumulo {@link TabletLocator} based on the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return an Accumulo tablet locator
+ * @throws TableNotFoundException
+ * if the table name set on the configuration doesn't exist
+ * @since 1.5.0
+ */
+ protected static TabletLocator getTabletLocator(JobConf job) throws TableNotFoundException {
+ return InputConfigurator.getTabletLocator(CLASS, job);
+ }
+
+ // InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext job)
+ /**
+ * Check whether a configuration is fully configured to be used with an Accumulo {@link org.apache.hadoop.mapreduce.InputFormat}.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @throws IOException
+ * if the context is improperly configured
+ * @since 1.5.0
+ */
+ protected static void validateOptions(JobConf job) throws IOException {
+ InputConfigurator.validateOptions(CLASS, job);
+ }
+
+ /**
+ * An abstract base class to be used to create {@link RecordReader} instances that convert from Accumulo {@link Key}/{@link Value} pairs to the user's K/V
+ * types.
+ *
+ * Subclasses must implement {@link #next(Object, Object)} to update key and value, and also to update the following variables:
+ * <ul>
+ * <li>Key {@link #currentKey} (used for progress reporting)</li>
+ * <li>int {@link #numKeysRead} (used for progress reporting)</li>
+ * </ul>
+ */
+ protected abstract static class RecordReaderBase<K,V> implements RecordReader<K,V> {
+ protected long numKeysRead;
+ protected Iterator<Entry<Key,Value>> scannerIterator;
+ protected org.apache.accumulo.core.client.mapred.RangeInputSplit split;
+
+ /**
+ * Apply the configured iterators from the configuration to the scanner.
+ *
+ * @param scanner
+ * the scanner to configure
+ */
+ protected void setupIterators(List<IteratorSetting> iterators, Scanner scanner) {
+ for (IteratorSetting iterator : iterators) {
+ scanner.addScanIterator(iterator);
+ }
+ }
+
+ /**
+ * Initialize a scanner over the given input split using this task attempt configuration.
+ */
+ public void initialize(InputSplit inSplit, JobConf job) throws IOException {
+ Scanner scanner;
+ split = (org.apache.accumulo.core.client.mapred.RangeInputSplit) inSplit;
+ log.debug("Initializing input split: " + split.getRange());
+
+ Instance instance = split.getInstance();
+ if (null == instance) {
+ instance = getInstance(job);
+ }
+
+ String principal = split.getPrincipal();
+ if (null == principal) {
+ principal = getPrincipal(job);
+ }
+
+ AuthenticationToken token = split.getToken();
+ if (null == token) {
+ String tokenClass = getTokenClass(job);
+ byte[] tokenBytes = getToken(job);
+ try {
+ token = CredentialHelper.extractToken(tokenClass, tokenBytes);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+ }
+
+ Authorizations authorizations = split.getAuths();
+ if (null == authorizations) {
+ authorizations = getScanAuthorizations(job);
+ }
+
+ String table = split.getTable();
+ if (null == table) {
+ table = getInputTableName(job);
+ }
+
+ Boolean isOffline = split.isOffline();
+ if (null == isOffline) {
+ isOffline = isOfflineScan(job);
+ }
+
+ Boolean isIsolated = split.isIsolatedScan();
+ if (null == isIsolated) {
+ isIsolated = isIsolated(job);
+ }
+
+ Boolean usesLocalIterators = split.usesLocalIterators();
+ if (null == usesLocalIterators) {
+ usesLocalIterators = usesLocalIterators(job);
+ }
+
+ List<IteratorSetting> iterators = split.getIterators();
+ if (null == iterators) {
+ iterators = getIterators(job);
+ }
+
+ Set<Pair<Text,Text>> columns = split.getFetchedColumns();
+ if (null == columns) {
+ columns = getFetchedColumns(job);
+ }
+
+ try {
+ log.debug("Creating connector with user: " + principal);
+ Connector conn = instance.getConnector(principal, token);
+ log.debug("Creating scanner for table: " + table);
+ log.debug("Authorizations are: " + authorizations);
+ if (isOffline) {
+ String tokenClass = token.getClass().getCanonicalName();
+ ByteBuffer tokenBuffer = ByteBuffer.wrap(CredentialHelper.toBytes(token));
+ scanner = new OfflineScanner(instance, new TCredentials(principal, tokenClass, tokenBuffer, instance.getInstanceID()), Tables.getTableId(instance,
+ table), authorizations);
+ } else {
+ scanner = conn.createScanner(table, authorizations);
+ }
+ if (isIsolated) {
+ log.info("Creating isolated scanner");
+ scanner = new IsolatedScanner(scanner);
+ }
+ if (usesLocalIterators) {
+ log.info("Using local iterators");
+ scanner = new ClientSideIteratorScanner(scanner);
+ }
+ setupIterators(iterators, scanner);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ // setup a scanner within the bounds of this split
+ for (Pair<Text,Text> c : columns) {
+ if (c.getSecond() != null) {
+ log.debug("Fetching column " + c.getFirst() + ":" + c.getSecond());
+ scanner.fetchColumn(c.getFirst(), c.getSecond());
+ } else {
+ log.debug("Fetching column family " + c.getFirst());
+ scanner.fetchColumnFamily(c.getFirst());
+ }
+ }
+
+ scanner.setRange(split.getRange());
+
+ numKeysRead = 0;
+
+ // do this last after setting all scanner options
+ scannerIterator = scanner.iterator();
+ }
+
+ @Override
+ public void close() {}
+
+ @Override
+ public long getPos() throws IOException {
+ return numKeysRead;
+ }
+
+ @Override
+ public float getProgress() throws IOException {
+ if (numKeysRead > 0 && currentKey == null)
+ return 1.0f;
+ return split.getProgress(currentKey);
+ }
+
+ protected Key currentKey = null;
+
+ }
+
+ Map<String,Map<KeyExtent,List<Range>>> binOfflineTable(JobConf job, String tableName, List<Range> ranges) throws TableNotFoundException, AccumuloException,
+ AccumuloSecurityException {
+
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+
+ Instance instance = getInstance(job);
+ Connector conn = instance.getConnector(getPrincipal(job), CredentialHelper.extractToken(getTokenClass(job), getToken(job)));
+ String tableId = Tables.getTableId(instance, tableName);
+
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ Tables.clearCache(instance);
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ throw new AccumuloException("Table is online " + tableName + "(" + tableId + ") cannot scan table in offline mode ");
+ }
+ }
+
+ for (Range range : ranges) {
+ Text startRow;
+
+ if (range.getStartKey() != null)
+ startRow = range.getStartKey().getRow();
+ else
+ startRow = new Text();
+
+ Range metadataRange = new Range(new KeyExtent(new Text(tableId), startRow, null).getMetadataEntry(), true, null, false);
+ Scanner scanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ Constants.METADATA_PREV_ROW_COLUMN.fetch(scanner);
+ scanner.fetchColumnFamily(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY);
+ scanner.setRange(metadataRange);
+
+ RowIterator rowIter = new RowIterator(scanner);
+
+ KeyExtent lastExtent = null;
+
+ while (rowIter.hasNext()) {
+ Iterator<Entry<Key,Value>> row = rowIter.next();
+ String last = "";
+ KeyExtent extent = null;
+ String location = null;
+
+ while (row.hasNext()) {
+ Entry<Key,Value> entry = row.next();
+ Key key = entry.getKey();
+
+ if (key.getColumnFamily().equals(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY)) {
+ last = entry.getValue().toString();
+ }
+
+ if (key.getColumnFamily().equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY)
+ || key.getColumnFamily().equals(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY)) {
+ location = entry.getValue().toString();
+ }
+
+ if (Constants.METADATA_PREV_ROW_COLUMN.hasColumns(key)) {
+ extent = new KeyExtent(key.getRow(), entry.getValue());
+ }
+
+ }
+
+ if (location != null)
+ return null;
+
+ if (!extent.getTableId().toString().equals(tableId)) {
+ throw new AccumuloException("Saw unexpected table Id " + tableId + " " + extent);
+ }
+
+ if (lastExtent != null && !extent.isPreviousExtent(lastExtent)) {
+ throw new AccumuloException(" " + lastExtent + " is not previous extent " + extent);
+ }
+
+ Map<KeyExtent,List<Range>> tabletRanges = binnedRanges.get(last);
+ if (tabletRanges == null) {
+ tabletRanges = new HashMap<KeyExtent,List<Range>>();
+ binnedRanges.put(last, tabletRanges);
+ }
+
+ List<Range> rangeList = tabletRanges.get(extent);
+ if (rangeList == null) {
+ rangeList = new ArrayList<Range>();
+ tabletRanges.put(extent, rangeList);
+ }
+
+ rangeList.add(range);
+
+ if (extent.getEndRow() == null || range.afterEndKey(new Key(extent.getEndRow()).followingKey(PartialKey.ROW))) {
+ break;
+ }
+
+ lastExtent = extent;
+ }
+
+ }
+
+ return binnedRanges;
+ }
+
+ /**
+ * Read the metadata table to get tablets and match up ranges to them.
+ */
+ @Override
+ public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
+ Level logLevel = getLogLevel(job);
+ log.setLevel(logLevel);
+
+ validateOptions(job);
+
+ String tableName = getInputTableName(job);
+ boolean autoAdjust = getAutoAdjustRanges(job);
+ List<Range> ranges = autoAdjust ? Range.mergeOverlapping(getRanges(job)) : getRanges(job);
+ Instance instance = getInstance(job);
+ boolean offline = isOfflineScan(job);
+ boolean isolated = isIsolated(job);
+ boolean localIterators = usesLocalIterators(job);
+ boolean mockInstance = (null != instance && MockInstance.class.equals(instance.getClass()));
+ Set<Pair<Text,Text>> fetchedColumns = getFetchedColumns(job);
+ Authorizations auths = getScanAuthorizations(job);
+ String principal = getPrincipal(job);
+ String tokenClass = getTokenClass(job);
+ byte[] tokenBytes = getToken(job);
+
+ AuthenticationToken token;
+ try {
+ token = CredentialHelper.extractToken(tokenClass, tokenBytes);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+
+ List<IteratorSetting> iterators = getIterators(job);
+
+ if (ranges.isEmpty()) {
+ ranges = new ArrayList<Range>(1);
+ ranges.add(new Range());
+ }
+
+ // get the metadata information for these ranges
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+ TabletLocator tl;
+ try {
+ if (isOfflineScan(job)) {
+ binnedRanges = binOfflineTable(job, tableName, ranges);
+ while (binnedRanges == null) {
+ // Some tablets were still online, try again
+ UtilWaitThread.sleep(100 + (int) (Math.random() * 100)); // sleep randomly between 100 and 200 ms
+ binnedRanges = binOfflineTable(job, tableName, ranges);
+ }
+ } else {
+ String tableId = null;
+ tl = getTabletLocator(job);
+ // its possible that the cache could contain complete, but old information about a tables tablets... so clear it
+ tl.invalidateCache();
+ while (!tl.binRanges(ranges, binnedRanges, new TCredentials(principal, tokenClass, ByteBuffer.wrap(tokenBytes), instance.getInstanceID())).isEmpty()) {
+ if (!(instance instanceof MockInstance)) {
+ if (tableId == null)
+ tableId = Tables.getTableId(instance, tableName);
+ if (!Tables.exists(instance, tableId))
+ throw new TableDeletedException(tableId);
+ if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
+ throw new TableOfflineException(instance, tableId);
+ }
+ binnedRanges.clear();
+ log.warn("Unable to locate bins for specified ranges. Retrying.");
+ UtilWaitThread.sleep(100 + (int) (Math.random() * 100)); // sleep randomly between 100 and 200 ms
+ tl.invalidateCache();
+ }
+ }
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ ArrayList<org.apache.accumulo.core.client.mapred.RangeInputSplit> splits = new ArrayList<org.apache.accumulo.core.client.mapred.RangeInputSplit>(
+ ranges.size());
+ HashMap<Range,ArrayList<String>> splitsToAdd = null;
+
+ if (!autoAdjust)
+ splitsToAdd = new HashMap<Range,ArrayList<String>>();
+
+ HashMap<String,String> hostNameCache = new HashMap<String,String>();
+
+ for (Entry<String,Map<KeyExtent,List<Range>>> tserverBin : binnedRanges.entrySet()) {
+ String ip = tserverBin.getKey().split(":", 2)[0];
+ String location = hostNameCache.get(ip);
+ if (location == null) {
+ InetAddress inetAddress = InetAddress.getByName(ip);
+ location = inetAddress.getHostName();
+ hostNameCache.put(ip, location);
+ }
+
+ for (Entry<KeyExtent,List<Range>> extentRanges : tserverBin.getValue().entrySet()) {
+ Range ke = extentRanges.getKey().toDataRange();
+ for (Range r : extentRanges.getValue()) {
+ if (autoAdjust) {
+ // divide ranges into smaller ranges, based on the tablets
+ splits.add(new org.apache.accumulo.core.client.mapred.RangeInputSplit(ke.clip(r), new String[] {location}));
+ } else {
+ // don't divide ranges
+ ArrayList<String> locations = splitsToAdd.get(r);
+ if (locations == null)
+ locations = new ArrayList<String>(1);
+ locations.add(location);
+ splitsToAdd.put(r, locations);
+ }
+ }
+ }
+ }
+
+ if (!autoAdjust)
+ for (Entry<Range,ArrayList<String>> entry : splitsToAdd.entrySet())
+ splits.add(new org.apache.accumulo.core.client.mapred.RangeInputSplit(entry.getKey(), entry.getValue().toArray(new String[0])));
+
+ for (org.apache.accumulo.core.client.mapred.RangeInputSplit split : splits) {
+ split.setTable(tableName);
+ split.setOffline(offline);
+ split.setIsolatedScan(isolated);
+ split.setUsesLocalIterators(localIterators);
+ split.setMockInstance(mockInstance);
+ split.setFetchedColumns(fetchedColumns);
+ split.setPrincipal(principal);
+ split.setToken(token);
+ split.setInstanceName(instance.getInstanceName());
+ split.setZooKeepers(instance.getZooKeepers());
+ split.setAuths(auths);
+ split.setIterators(iterators);
+ split.setLogLevel(logLevel);
+ }
+
+ return splits.toArray(new InputSplit[splits.size()]);
+ }
+
+ /**
+ * @deprecated since 1.5.2; Use {@link org.apache.accumulo.core.client.mapred.RangeInputSplit} instead.
+ * @see org.apache.accumulo.core.client.mapred.RangeInputSplit
+ */
+ @Deprecated
+ public static class RangeInputSplit extends org.apache.accumulo.core.client.mapred.RangeInputSplit {
+ public RangeInputSplit() {
+ super();
+ }
+
+ public RangeInputSplit(Range range, String[] locations) {
+ super(range, locations);
+ }
+ }
+}
[46/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
index cb54856,0000000..1a029dc
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
@@@ -1,273 -1,0 +1,272 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce.lib.util;
+
+import java.nio.charset.Charset;
+
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * @since 1.5.0
+ */
+public class ConfiguratorBase {
+
+ /**
+ * Configuration keys for {@link Instance#getConnector(String, AuthenticationToken)}.
+ *
+ * @since 1.5.0
+ */
+ public static enum ConnectorInfo {
+ IS_CONFIGURED, PRINCIPAL, TOKEN, TOKEN_CLASS
+ }
+
+ /**
+ * Configuration keys for {@link Instance}, {@link ZooKeeperInstance}, and {@link MockInstance}.
+ *
+ * @since 1.5.0
+ */
+ protected static enum InstanceOpts {
+ TYPE, NAME, ZOO_KEEPERS;
+ }
+
+ /**
+ * Configuration keys for general configuration options.
+ *
+ * @since 1.5.0
+ */
+ protected static enum GeneralOpts {
+ LOG_LEVEL
+ }
+
+ /**
+ * Provides a configuration key for a given feature enum, prefixed by the implementingClass
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param e
+ * the enum used to provide the unique part of the configuration key
+ * @return the configuration key
+ * @since 1.5.0
+ */
+ protected static String enumToConfKey(Class<?> implementingClass, Enum<?> e) {
+ return implementingClass.getSimpleName() + "." + e.getDeclaringClass().getSimpleName() + "." + StringUtils.camelize(e.name().toLowerCase());
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
+ * conversion to a string, and is not intended to be secure.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param principal
+ * a valid Accumulo user name
+ * @param token
+ * the user's password
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+ public static void setConnectorInfo(Class<?> implementingClass, Configuration conf, String principal, AuthenticationToken token)
+ throws AccumuloSecurityException {
+ if (isConnectorInfoSet(implementingClass, conf))
+ throw new IllegalStateException("Connector info for " + implementingClass.getSimpleName() + " can only be set once per job");
+
+ ArgumentChecker.notNull(principal, token);
+ conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL), principal);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN_CLASS), token.getClass().getCanonicalName());
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN), CredentialHelper.tokenAsBase64(token));
+ }
+
+ /**
+ * Determines if the connector info has already been set for this instance.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the connector info has already been set, false otherwise
+ * @since 1.5.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static Boolean isConnectorInfoSet(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), false);
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the principal
+ * @since 1.5.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static String getPrincipal(Class<?> implementingClass, Configuration conf) {
+ return conf.get(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL));
+ }
+
+ /**
+ * Gets the serialized token class from the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the principal
+ * @since 1.5.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static String getTokenClass(Class<?> implementingClass, Configuration conf) {
+ return conf.get(enumToConfKey(implementingClass, ConnectorInfo.TOKEN_CLASS));
+ }
+
+ /**
+ * Gets the password from the configuration. WARNING: The password is stored in the Configuration and shared with all MapReduce tasks; It is BASE64 encoded to
+ * provide a charset safe conversion to a string, and is not intended to be secure.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the decoded principal's authentication token
+ * @since 1.5.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static byte[] getToken(Class<?> implementingClass, Configuration conf) {
+ return Base64.decodeBase64(conf.get(enumToConfKey(implementingClass, ConnectorInfo.TOKEN), "").getBytes(Charset.forName("UTF-8")));
+ }
+
+ /**
+ * Configures a {@link ZooKeeperInstance} for this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param instanceName
+ * the Accumulo instance name
+ * @param zooKeepers
+ * a comma-separated list of zookeeper servers
+ * @since 1.5.0
+ */
+ public static void setZooKeeperInstance(Class<?> implementingClass, Configuration conf, String instanceName, String zooKeepers) {
+ String key = enumToConfKey(implementingClass, InstanceOpts.TYPE);
+ if (!conf.get(key, "").isEmpty())
+ throw new IllegalStateException("Instance info can only be set once per job; it has already been configured with " + conf.get(key));
+ conf.set(key, "ZooKeeperInstance");
+
+ ArgumentChecker.notNull(instanceName, zooKeepers);
+ conf.set(enumToConfKey(implementingClass, InstanceOpts.NAME), instanceName);
+ conf.set(enumToConfKey(implementingClass, InstanceOpts.ZOO_KEEPERS), zooKeepers);
+ }
+
+ /**
+ * Configures a {@link MockInstance} for this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param instanceName
+ * the Accumulo instance name
+ * @since 1.5.0
+ */
+ public static void setMockInstance(Class<?> implementingClass, Configuration conf, String instanceName) {
+ String key = enumToConfKey(implementingClass, InstanceOpts.TYPE);
+ if (!conf.get(key, "").isEmpty())
+ throw new IllegalStateException("Instance info can only be set once per job; it has already been configured with " + conf.get(key));
+ conf.set(key, "MockInstance");
+
+ ArgumentChecker.notNull(instanceName);
+ conf.set(enumToConfKey(implementingClass, InstanceOpts.NAME), instanceName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link Instance} based on the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return an Accumulo instance
+ * @since 1.5.0
+ * @see #setZooKeeperInstance(Class, Configuration, String, String)
+ * @see #setMockInstance(Class, Configuration, String)
+ */
+ public static Instance getInstance(Class<?> implementingClass, Configuration conf) {
+ String instanceType = conf.get(enumToConfKey(implementingClass, InstanceOpts.TYPE), "");
+ if ("MockInstance".equals(instanceType))
+ return new MockInstance(conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME)));
+ else if ("ZooKeeperInstance".equals(instanceType)) {
+ return new ZooKeeperInstance(conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME)), conf.get(enumToConfKey(implementingClass,
+ InstanceOpts.ZOO_KEEPERS)));
+ } else if (instanceType.isEmpty())
+ throw new IllegalStateException("Instance has not been configured for " + implementingClass.getSimpleName());
+ else
+ throw new IllegalStateException("Unrecognized instance type " + instanceType);
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param level
+ * the logging level
+ * @since 1.5.0
+ */
+ public static void setLogLevel(Class<?> implementingClass, Configuration conf, Level level) {
+ ArgumentChecker.notNull(level);
+ Logger.getLogger(implementingClass).setLevel(level);
+ conf.setInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), level.toInt());
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the log level
+ * @since 1.5.0
+ * @see #setLogLevel(Class, Configuration, Level)
+ */
+ public static Level getLogLevel(Class<?> implementingClass, Configuration conf) {
+ return Level.toLevel(conf.getInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), Level.INFO.toInt()));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/mock/MockBatchDeleter.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mock/MockBatchDeleter.java
index 67362a2,0000000..6f321ff
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockBatchDeleter.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockBatchDeleter.java
@@@ -1,77 -1,0 +1,73 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mock;
+
+import java.util.Iterator;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.BatchDeleter;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
+
+/**
+ * {@link BatchDeleter} for a {@link MockAccumulo} instance. Behaves similarly to a regular {@link BatchDeleter}, with a few exceptions:
+ * <ol>
+ * <li>There is no waiting for memory to fill before flushing</li>
+ * <li>Only one thread is used for writing</li>
+ * </ol>
+ *
+ * Otherwise, it behaves as expected.
+ */
+public class MockBatchDeleter extends MockBatchScanner implements BatchDeleter {
+
+ private final MockAccumulo acc;
+ private final String tableName;
+
+ /**
+ * Create a {@link BatchDeleter} for the specified instance on the specified table where the writer uses the specified {@link Authorizations}.
- *
- * @param acc
- * @param tableName
- * @param auths
+ */
+ public MockBatchDeleter(MockAccumulo acc, String tableName, Authorizations auths) {
+ super(acc.tables.get(tableName), auths);
+ this.acc = acc;
+ this.tableName = tableName;
+ }
+
+ @Override
+ public void delete() throws MutationsRejectedException, TableNotFoundException {
+
+ BatchWriter writer = new MockBatchWriter(acc, tableName);
+ try {
+ Iterator<Entry<Key,Value>> iter = super.iterator();
+ while (iter.hasNext()) {
+ Entry<Key,Value> next = iter.next();
+ Key k = next.getKey();
+ Mutation m = new Mutation(k.getRow());
+ m.putDelete(k.getColumnFamily(), k.getColumnQualifier(), new ColumnVisibility(k.getColumnVisibility()), k.getTimestamp());
+ writer.addMutation(m);
+ }
+ } finally {
+ writer.close();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java
index 34eb3de,0000000..cb9481f
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java
@@@ -1,133 -1,0 +1,90 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mock;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.admin.ActiveCompaction;
+import org.apache.accumulo.core.client.admin.ActiveScan;
+import org.apache.accumulo.core.client.admin.InstanceOperations;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+
+/**
+ *
+ */
+public class MockInstanceOperations implements InstanceOperations {
+ MockAccumulo acu;
+
- /**
- * @param acu
- */
+ public MockInstanceOperations(MockAccumulo acu) {
+ this.acu = acu;
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#setProperty(java.lang.String, java.lang.String)
- */
+ @Override
+ public void setProperty(String property, String value) throws AccumuloException, AccumuloSecurityException {
+ acu.setProperty(property, value);
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#removeProperty(java.lang.String)
- */
+ @Override
+ public void removeProperty(String property) throws AccumuloException, AccumuloSecurityException {
+ acu.removeProperty(property);
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getSystemConfiguration()
- */
+ @Override
+ public Map<String,String> getSystemConfiguration() throws AccumuloException, AccumuloSecurityException {
+ return acu.systemProperties;
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getSiteConfiguration()
- */
+ @Override
+ public Map<String,String> getSiteConfiguration() throws AccumuloException, AccumuloSecurityException {
+ return acu.systemProperties;
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getTabletServers()
- */
+ @Override
+ public List<String> getTabletServers() {
+ return new ArrayList<String>();
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getActiveScans(java.lang.String)
- */
+ @Override
+ public List<ActiveScan> getActiveScans(String tserver) throws AccumuloException, AccumuloSecurityException {
+ return new ArrayList<ActiveScan>();
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#testClassLoad(java.lang.String, java.lang.String)
- */
+ @Override
+ public boolean testClassLoad(String className, String asTypeName) throws AccumuloException, AccumuloSecurityException {
+ try {
+ AccumuloVFSClassLoader.loadClass(className, Class.forName(asTypeName));
+ } catch (ClassNotFoundException e) {
+ e.printStackTrace();
+ return false;
+ }
+ return true;
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.client.admin.InstanceOperations#getActiveCompactions(java.lang.String)
- */
+ @Override
+ public List<ActiveCompaction> getActiveCompactions(String tserver) throws AccumuloException, AccumuloSecurityException {
+ return new ArrayList<ActiveCompaction>();
+ }
+
+ @Override
+ public void ping(String tserver) throws AccumuloException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/data/ColumnUpdate.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/ColumnUpdate.java
index bfba00f,0000000..78f2d15
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/data/ColumnUpdate.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/ColumnUpdate.java
@@@ -1,110 -1,0 +1,109 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
+import java.util.Arrays;
+
+/**
+ * A single column and value pair within a mutation
+ *
+ */
+
+public class ColumnUpdate {
+
+ private byte[] columnFamily;
+ private byte[] columnQualifier;
+ private byte[] columnVisibility;
+ private long timestamp;
+ private boolean hasTimestamp;
+ private byte[] val;
+ private boolean deleted;
+
+ public ColumnUpdate(byte[] cf, byte[] cq, byte[] cv, boolean hasts, long ts, boolean deleted, byte[] val) {
+ this.columnFamily = cf;
+ this.columnQualifier = cq;
+ this.columnVisibility = cv;
+ this.hasTimestamp = hasts;
+ this.timestamp = ts;
+ this.deleted = deleted;
+ this.val = val;
+ }
+
+ /**
+ * @deprecated use setTimestamp(long);
- * @param timestamp
+ */
+ @Deprecated
+ public void setSystemTimestamp(long timestamp) {
+ if (hasTimestamp)
+ throw new IllegalStateException("Cannot set system timestamp when user set a timestamp");
+ }
+
+ public boolean hasTimestamp() {
+ return hasTimestamp;
+ }
+
+ /**
+ * Returns the column
+ *
+ */
+ public byte[] getColumnFamily() {
+ return columnFamily;
+ }
+
+ public byte[] getColumnQualifier() {
+ return columnQualifier;
+ }
+
+ public byte[] getColumnVisibility() {
+ return columnVisibility;
+ }
+
+ public long getTimestamp() {
+ return this.timestamp;
+ }
+
+ public boolean isDeleted() {
+ return this.deleted;
+ }
+
+ public byte[] getValue() {
+ return this.val;
+ }
+
+ @Override
+ public String toString() {
+ return Arrays.toString(columnFamily) + ":" + Arrays.toString(columnQualifier) + " ["
+ + Arrays.toString(columnVisibility) + "] " + (hasTimestamp ? timestamp : "NO_TIME_STAMP") + " " + Arrays.toString(val) + " " + deleted;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof ColumnUpdate))
+ return false;
+ ColumnUpdate upd = (ColumnUpdate) obj;
+ return Arrays.equals(getColumnFamily(), upd.getColumnFamily()) && Arrays.equals(getColumnQualifier(), upd.getColumnQualifier())
+ && Arrays.equals(getColumnVisibility(), upd.getColumnVisibility()) && isDeleted() == upd.isDeleted() && Arrays.equals(getValue(), upd.getValue())
+ && hasTimestamp() == upd.hasTimestamp() && getTimestamp() == upd.getTimestamp();
+ }
+
+ @Override
+ public int hashCode() {
+ return Arrays.hashCode(columnFamily) + Arrays.hashCode(columnQualifier) + Arrays.hashCode(columnVisibility)
+ + (hasTimestamp ? (Boolean.TRUE.hashCode() + Long.valueOf(timestamp).hashCode()) : Boolean.FALSE.hashCode())
+ + (deleted ? Boolean.TRUE.hashCode() : (Boolean.FALSE.hashCode() + Arrays.hashCode(val)));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/data/Key.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Key.java
index 4b6867f,0000000..2b44359
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/data/Key.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Key.java
@@@ -1,863 -1,0 +1,864 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
+/**
+ * This is the Key used to store and access individual values in Accumulo. A Key is a tuple composed of a row, column family, column qualifier,
+ * column visibility, timestamp, and delete marker.
+ *
+ * Keys are comparable and therefore have a sorted order defined by {@link #compareTo(Key)}.
+ *
+ */
+
+import static org.apache.accumulo.core.util.ByteBufferUtil.toBytes;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.thrift.TKey;
+import org.apache.accumulo.core.data.thrift.TKeyValue;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.io.WritableUtils;
+
+public class Key implements WritableComparable<Key>, Cloneable {
+
+ protected byte[] row;
+ protected byte[] colFamily;
+ protected byte[] colQualifier;
+ protected byte[] colVisibility;
+ protected long timestamp;
+ protected boolean deleted;
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof Key)
+ return this.equals((Key) o, PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME_DEL);
+ return false;
+ }
+
+ private static final byte EMPTY_BYTES[] = new byte[0];
+
+ private byte[] copyIfNeeded(byte ba[], int off, int len, boolean copyData) {
+ if (len == 0)
+ return EMPTY_BYTES;
+
+ if (!copyData && ba.length == len && off == 0)
+ return ba;
+
+ byte[] copy = new byte[len];
+ System.arraycopy(ba, off, copy, 0, len);
+ return copy;
+ }
+
+ private final void init(byte r[], int rOff, int rLen, byte cf[], int cfOff, int cfLen, byte cq[], int cqOff, int cqLen, byte cv[], int cvOff, int cvLen,
+ long ts, boolean del, boolean copy) {
+ row = copyIfNeeded(r, rOff, rLen, copy);
+ colFamily = copyIfNeeded(cf, cfOff, cfLen, copy);
+ colQualifier = copyIfNeeded(cq, cqOff, cqLen, copy);
+ colVisibility = copyIfNeeded(cv, cvOff, cvLen, copy);
+ timestamp = ts;
+ deleted = del;
+ }
+
+ /**
+ * Creates a key with empty row, empty column family, empty column qualifier, empty column visibility, timestamp {@link Long#MAX_VALUE}, and delete marker
+ * false.
+ */
+ public Key() {
+ row = EMPTY_BYTES;
+ colFamily = EMPTY_BYTES;
+ colQualifier = EMPTY_BYTES;
+ colVisibility = EMPTY_BYTES;
+ timestamp = Long.MAX_VALUE;
+ deleted = false;
+ }
+
+ /**
+ * Creates a key with the specified row, empty column family, empty column qualifier, empty column visibility, timestamp {@link Long#MAX_VALUE}, and delete
+ * marker false.
+ */
+ public Key(Text row) {
+ init(row.getBytes(), 0, row.getLength(), EMPTY_BYTES, 0, 0, EMPTY_BYTES, 0, 0, EMPTY_BYTES, 0, 0, Long.MAX_VALUE, false, true);
+ }
+
+ /**
+ * Creates a key with the specified row, empty column family, empty column qualifier, empty column visibility, the specified timestamp, and delete marker
+ * false.
+ */
+ public Key(Text row, long ts) {
+ this(row);
+ timestamp = ts;
+ }
+
+ public Key(byte row[], int rOff, int rLen, byte cf[], int cfOff, int cfLen, byte cq[], int cqOff, int cqLen, byte cv[], int cvOff, int cvLen, long ts) {
+ init(row, rOff, rLen, cf, cfOff, cfLen, cq, cqOff, cqLen, cv, cvOff, cvLen, ts, false, true);
+ }
+
+ public Key(byte[] row, byte[] colFamily, byte[] colQualifier, byte[] colVisibility, long timestamp) {
+ this(row, colFamily, colQualifier, colVisibility, timestamp, false, true);
+ }
+
+ public Key(byte[] row, byte[] cf, byte[] cq, byte[] cv, long ts, boolean deleted) {
+ this(row, cf, cq, cv, ts, deleted, true);
+ }
+
+ public Key(byte[] row, byte[] cf, byte[] cq, byte[] cv, long ts, boolean deleted, boolean copy) {
+ init(row, 0, row.length, cf, 0, cf.length, cq, 0, cq.length, cv, 0, cv.length, ts, deleted, copy);
+ }
+
+ /**
+ * Creates a key with the specified row, the specified column family, empty column qualifier, empty column visibility, timestamp {@link Long#MAX_VALUE}, and
+ * delete marker false.
+ */
+ public Key(Text row, Text cf) {
+ init(row.getBytes(), 0, row.getLength(), cf.getBytes(), 0, cf.getLength(), EMPTY_BYTES, 0, 0, EMPTY_BYTES, 0, 0, Long.MAX_VALUE, false, true);
+ }
+
+ /**
+ * Creates a key with the specified row, the specified column family, the specified column qualifier, empty column visibility, timestamp
+ * {@link Long#MAX_VALUE}, and delete marker false.
+ */
+ public Key(Text row, Text cf, Text cq) {
+ init(row.getBytes(), 0, row.getLength(), cf.getBytes(), 0, cf.getLength(), cq.getBytes(), 0, cq.getLength(), EMPTY_BYTES, 0, 0, Long.MAX_VALUE, false, true);
+ }
+
+ /**
+ * Creates a key with the specified row, the specified column family, the specified column qualifier, the specified column visibility, timestamp
+ * {@link Long#MAX_VALUE}, and delete marker false.
+ */
+ public Key(Text row, Text cf, Text cq, Text cv) {
+ init(row.getBytes(), 0, row.getLength(), cf.getBytes(), 0, cf.getLength(), cq.getBytes(), 0, cq.getLength(), cv.getBytes(), 0, cv.getLength(),
+ Long.MAX_VALUE, false, true);
+ }
+
+ /**
+ * Creates a key with the specified row, the specified column family, the specified column qualifier, empty column visibility, the specified timestamp, and
+ * delete marker false.
+ */
+ public Key(Text row, Text cf, Text cq, long ts) {
+ init(row.getBytes(), 0, row.getLength(), cf.getBytes(), 0, cf.getLength(), cq.getBytes(), 0, cq.getLength(), EMPTY_BYTES, 0, 0, ts, false, true);
+ }
+
+ /**
+ * Creates a key with the specified row, the specified column family, the specified column qualifier, the specified column visibility, the specified
+ * timestamp, and delete marker false.
+ */
+ public Key(Text row, Text cf, Text cq, Text cv, long ts) {
+ init(row.getBytes(), 0, row.getLength(), cf.getBytes(), 0, cf.getLength(), cq.getBytes(), 0, cq.getLength(), cv.getBytes(), 0, cv.getLength(), ts, false,
+ true);
+ }
+
+ /**
+ * Creates a key with the specified row, the specified column family, the specified column qualifier, the specified column visibility, the specified
+ * timestamp, and delete marker false.
+ */
+ public Key(Text row, Text cf, Text cq, ColumnVisibility cv, long ts) {
+ byte[] expr = cv.getExpression();
+ init(row.getBytes(), 0, row.getLength(), cf.getBytes(), 0, cf.getLength(), cq.getBytes(), 0, cq.getLength(), expr, 0, expr.length, ts, false, true);
+ }
+
+ /**
+ * Converts CharSequence to Text and creates a Key using {@link #Key(Text)}.
+ */
+ public Key(CharSequence row) {
+ this(new Text(row.toString()));
+ }
+
+ /**
+ * Converts CharSequence to Text and creates a Key using {@link #Key(Text,Text)}.
+ */
+ public Key(CharSequence row, CharSequence cf) {
+ this(new Text(row.toString()), new Text(cf.toString()));
+ }
+
+ /**
+ * Converts CharSequence to Text and creates a Key using {@link #Key(Text,Text,Text)}.
+ */
+ public Key(CharSequence row, CharSequence cf, CharSequence cq) {
+ this(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()));
+ }
+
+ /**
+ * Converts CharSequence to Text and creates a Key using {@link #Key(Text,Text,Text,Text)}.
+ */
+ public Key(CharSequence row, CharSequence cf, CharSequence cq, CharSequence cv) {
+ this(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()), new Text(cv.toString()));
+ }
+
+ /**
+ * Converts CharSequence to Text and creates a Key using {@link #Key(Text,Text,Text,long)}.
+ */
+ public Key(CharSequence row, CharSequence cf, CharSequence cq, long ts) {
+ this(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()), ts);
+ }
+
+ /**
+ * Converts CharSequence to Text and creates a Key using {@link #Key(Text,Text,Text,Text,long)}.
+ */
+ public Key(CharSequence row, CharSequence cf, CharSequence cq, CharSequence cv, long ts) {
+ this(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()), new Text(cv.toString()), ts);
+ }
+
+ /**
+ * Converts CharSequence to Text and creates a Key using {@link #Key(Text,Text,Text,ColumnVisibility,long)}.
+ */
+ public Key(CharSequence row, CharSequence cf, CharSequence cq, ColumnVisibility cv, long ts) {
+ this(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()), new Text(cv.getExpression()), ts);
+ }
+
+ private byte[] followingArray(byte ba[]) {
+ byte[] fba = new byte[ba.length + 1];
+ System.arraycopy(ba, 0, fba, 0, ba.length);
+ fba[ba.length] = (byte) 0x00;
+ return fba;
+ }
+
+ /**
+ * Returns a key that will sort immediately after this key.
+ *
+ * @param part
+ * PartialKey except {@link PartialKey#ROW_COLFAM_COLQUAL_COLVIS_TIME_DEL}
+ */
+ public Key followingKey(PartialKey part) {
+ Key returnKey = new Key();
+ switch (part) {
+ case ROW:
+ returnKey.row = followingArray(row);
+ break;
+ case ROW_COLFAM:
+ returnKey.row = row;
+ returnKey.colFamily = followingArray(colFamily);
+ break;
+ case ROW_COLFAM_COLQUAL:
+ returnKey.row = row;
+ returnKey.colFamily = colFamily;
+ returnKey.colQualifier = followingArray(colQualifier);
+ break;
+ case ROW_COLFAM_COLQUAL_COLVIS:
+ // This isn't useful for inserting into accumulo, but may be useful for lookups.
+ returnKey.row = row;
+ returnKey.colFamily = colFamily;
+ returnKey.colQualifier = colQualifier;
+ returnKey.colVisibility = followingArray(colVisibility);
+ break;
+ case ROW_COLFAM_COLQUAL_COLVIS_TIME:
+ returnKey.row = row;
+ returnKey.colFamily = colFamily;
+ returnKey.colQualifier = colQualifier;
+ returnKey.colVisibility = colVisibility;
+ returnKey.setTimestamp(timestamp - 1);
+ returnKey.deleted = false;
+ break;
+ default:
+ throw new IllegalArgumentException("Partial key specification " + part + " disallowed");
+ }
+ return returnKey;
+ }
+
+ /**
+ * Creates a key with the same row, column family, column qualifier, column visibility, timestamp, and delete marker as the given key.
+ */
+ public Key(Key other) {
+ set(other);
+ }
+
+ public Key(TKey tkey) {
+ this.row = toBytes(tkey.row);
+ this.colFamily = toBytes(tkey.colFamily);
+ this.colQualifier = toBytes(tkey.colQualifier);
+ this.colVisibility = toBytes(tkey.colVisibility);
+ this.timestamp = tkey.timestamp;
+ this.deleted = false;
+
+ if (row == null) {
+ throw new IllegalArgumentException("null row");
+ }
+ if (colFamily == null) {
+ throw new IllegalArgumentException("null column family");
+ }
+ if (colQualifier == null) {
+ throw new IllegalArgumentException("null column qualifier");
+ }
+ if (colVisibility == null) {
+ throw new IllegalArgumentException("null column visibility");
+ }
+ }
+
+ /**
+ * This method gives users control over allocation of Text objects by copying into the passed in text.
+ *
+ * @param r
+ * the key's row will be copied into this Text
+ * @return the Text that was passed in
+ */
+
+ public Text getRow(Text r) {
+ r.set(row, 0, row.length);
+ return r;
+ }
+
+ /**
+ * This method returns a pointer to the keys internal data and does not copy it.
+ *
+ * @return ByteSequence that points to the internal key row data.
+ */
+
+ public ByteSequence getRowData() {
+ return new ArrayByteSequence(row);
+ }
+
+ /**
+ * This method allocates a Text object and copies into it.
+ *
+ * @return Text containing the row field
+ */
+
+ public Text getRow() {
+ return getRow(new Text());
+ }
+
+ /**
+ * Efficiently compare the the row of a key w/o allocating a text object and copying the row into it.
+ *
+ * @param r
+ * row to compare to keys row
+ * @return same as {@link #getRow()}.compareTo(r)
+ */
+
+ public int compareRow(Text r) {
+ return WritableComparator.compareBytes(row, 0, row.length, r.getBytes(), 0, r.getLength());
+ }
+
+ /**
+ * This method returns a pointer to the keys internal data and does not copy it.
+ *
+ * @return ByteSequence that points to the internal key column family data.
+ */
+
+ public ByteSequence getColumnFamilyData() {
+ return new ArrayByteSequence(colFamily);
+ }
+
+ /**
+ * This method gives users control over allocation of Text objects by copying into the passed in text.
+ *
+ * @param cf
+ * the key's column family will be copied into this Text
+ * @return the Text that was passed in
+ */
+
+ public Text getColumnFamily(Text cf) {
+ cf.set(colFamily, 0, colFamily.length);
+ return cf;
+ }
+
+ /**
+ * This method allocates a Text object and copies into it.
+ *
+ * @return Text containing the column family field
+ */
+
+ public Text getColumnFamily() {
+ return getColumnFamily(new Text());
+ }
+
+ /**
+ * Efficiently compare the the column family of a key w/o allocating a text object and copying the column qualifier into it.
+ *
+ * @param cf
+ * column family to compare to keys column family
+ * @return same as {@link #getColumnFamily()}.compareTo(cf)
+ */
+
+ public int compareColumnFamily(Text cf) {
+ return WritableComparator.compareBytes(colFamily, 0, colFamily.length, cf.getBytes(), 0, cf.getLength());
+ }
+
+ /**
+ * This method returns a pointer to the keys internal data and does not copy it.
+ *
+ * @return ByteSequence that points to the internal key column qualifier data.
+ */
+
+ public ByteSequence getColumnQualifierData() {
+ return new ArrayByteSequence(colQualifier);
+ }
+
+ /**
+ * This method gives users control over allocation of Text objects by copying into the passed in text.
+ *
+ * @param cq
+ * the key's column qualifier will be copied into this Text
+ * @return the Text that was passed in
+ */
+
+ public Text getColumnQualifier(Text cq) {
+ cq.set(colQualifier, 0, colQualifier.length);
+ return cq;
+ }
+
+ /**
+ * This method allocates a Text object and copies into it.
+ *
+ * @return Text containing the column qualifier field
+ */
+
+ public Text getColumnQualifier() {
+ return getColumnQualifier(new Text());
+ }
+
+ /**
+ * Efficiently compare the the column qualifier of a key w/o allocating a text object and copying the column qualifier into it.
+ *
+ * @param cq
+ * column family to compare to keys column qualifier
+ * @return same as {@link #getColumnQualifier()}.compareTo(cq)
+ */
+
+ public int compareColumnQualifier(Text cq) {
+ return WritableComparator.compareBytes(colQualifier, 0, colQualifier.length, cq.getBytes(), 0, cq.getLength());
+ }
+
+ public void setTimestamp(long ts) {
+ this.timestamp = ts;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public boolean isDeleted() {
+ return deleted;
+ }
+
+ public void setDeleted(boolean deleted) {
+ this.deleted = deleted;
+ }
+
+ /**
+ * This method returns a pointer to the keys internal data and does not copy it.
+ *
+ * @return ByteSequence that points to the internal key column visibility data.
+ */
+
+ public ByteSequence getColumnVisibilityData() {
+ return new ArrayByteSequence(colVisibility);
+ }
+
+ /**
+ * This method allocates a Text object and copies into it.
+ *
+ * @return Text containing the column visibility field
+ */
+
+ public final Text getColumnVisibility() {
+ return getColumnVisibility(new Text());
+ }
+
+ /**
+ * This method gives users control over allocation of Text objects by copying into the passed in text.
+ *
+ * @param cv
+ * the key's column visibility will be copied into this Text
+ * @return the Text that was passed in
+ */
+
+ public final Text getColumnVisibility(Text cv) {
+ cv.set(colVisibility, 0, colVisibility.length);
+ return cv;
+ }
+
+ /**
+ * This method creates a new ColumnVisibility representing the column visibility for this key
+ *
+ * WARNING: using this method may inhibit performance since a new ColumnVisibility object is created on every call.
+ *
+ * @return A new object representing the column visibility field
+ * @since 1.5.0
+ */
+ public final ColumnVisibility getColumnVisibilityParsed() {
+ return new ColumnVisibility(colVisibility);
+ }
+
+ /**
+ * Sets this key's row, column family, column qualifier, column visibility, timestamp, and delete marker to be the same as another key's.
+ */
+ public void set(Key k) {
+ row = k.row;
+ colFamily = k.colFamily;
+ colQualifier = k.colQualifier;
+ colVisibility = k.colVisibility;
+ timestamp = k.timestamp;
+ deleted = k.deleted;
+
+ }
+
++ @Override
+ public void readFields(DataInput in) throws IOException {
+ // this method is a little screwy so it will be compatible with older
+ // code that serialized data
+
+ int colFamilyOffset = WritableUtils.readVInt(in);
+ int colQualifierOffset = WritableUtils.readVInt(in);
+ int colVisibilityOffset = WritableUtils.readVInt(in);
+ int totalLen = WritableUtils.readVInt(in);
+
+ row = new byte[colFamilyOffset];
+ colFamily = new byte[colQualifierOffset - colFamilyOffset];
+ colQualifier = new byte[colVisibilityOffset - colQualifierOffset];
+ colVisibility = new byte[totalLen - colVisibilityOffset];
+
+ in.readFully(row);
+ in.readFully(colFamily);
+ in.readFully(colQualifier);
+ in.readFully(colVisibility);
+
+ timestamp = WritableUtils.readVLong(in);
+ deleted = in.readBoolean();
+ }
+
++ @Override
+ public void write(DataOutput out) throws IOException {
+
+ int colFamilyOffset = row.length;
+ int colQualifierOffset = colFamilyOffset + colFamily.length;
+ int colVisibilityOffset = colQualifierOffset + colQualifier.length;
+ int totalLen = colVisibilityOffset + colVisibility.length;
+
+ WritableUtils.writeVInt(out, colFamilyOffset);
+ WritableUtils.writeVInt(out, colQualifierOffset);
+ WritableUtils.writeVInt(out, colVisibilityOffset);
+
+ WritableUtils.writeVInt(out, totalLen);
+
+ out.write(row);
+ out.write(colFamily);
+ out.write(colQualifier);
+ out.write(colVisibility);
+
+ WritableUtils.writeVLong(out, timestamp);
+ out.writeBoolean(deleted);
+ }
+
+ /**
+ * Compare part of a key. For example compare just the row and column family, and if those are equal then return true.
+ *
+ */
+
+ public boolean equals(Key other, PartialKey part) {
+ switch (part) {
+ case ROW:
+ return isEqual(row, other.row);
+ case ROW_COLFAM:
+ return isEqual(row, other.row) && isEqual(colFamily, other.colFamily);
+ case ROW_COLFAM_COLQUAL:
+ return isEqual(row, other.row) && isEqual(colFamily, other.colFamily) && isEqual(colQualifier, other.colQualifier);
+ case ROW_COLFAM_COLQUAL_COLVIS:
+ return isEqual(row, other.row) && isEqual(colFamily, other.colFamily) && isEqual(colQualifier, other.colQualifier)
+ && isEqual(colVisibility, other.colVisibility);
+ case ROW_COLFAM_COLQUAL_COLVIS_TIME:
+ return isEqual(row, other.row) && isEqual(colFamily, other.colFamily) && isEqual(colQualifier, other.colQualifier)
+ && isEqual(colVisibility, other.colVisibility) && timestamp == other.timestamp;
+ case ROW_COLFAM_COLQUAL_COLVIS_TIME_DEL:
+ return isEqual(row, other.row) && isEqual(colFamily, other.colFamily) && isEqual(colQualifier, other.colQualifier)
+ && isEqual(colVisibility, other.colVisibility) && timestamp == other.timestamp && deleted == other.deleted;
+ default:
+ throw new IllegalArgumentException("Unrecognized partial key specification " + part);
+ }
+ }
+
+ /**
+ * Compare elements of a key given by a {@link PartialKey}. For example, for {@link PartialKey#ROW_COLFAM}, compare just the row and column family. If the
+ * rows are not equal, return the result of the row comparison; otherwise, return the result of the column family comparison.
+ *
+ * @see #compareTo(Key)
+ */
+
+ public int compareTo(Key other, PartialKey part) {
+ // check for matching row
+ int result = WritableComparator.compareBytes(row, 0, row.length, other.row, 0, other.row.length);
+ if (result != 0 || part.equals(PartialKey.ROW))
+ return result;
+
+ // check for matching column family
+ result = WritableComparator.compareBytes(colFamily, 0, colFamily.length, other.colFamily, 0, other.colFamily.length);
+ if (result != 0 || part.equals(PartialKey.ROW_COLFAM))
+ return result;
+
+ // check for matching column qualifier
+ result = WritableComparator.compareBytes(colQualifier, 0, colQualifier.length, other.colQualifier, 0, other.colQualifier.length);
+ if (result != 0 || part.equals(PartialKey.ROW_COLFAM_COLQUAL))
+ return result;
+
+ // check for matching column visibility
+ result = WritableComparator.compareBytes(colVisibility, 0, colVisibility.length, other.colVisibility, 0, other.colVisibility.length);
+ if (result != 0 || part.equals(PartialKey.ROW_COLFAM_COLQUAL_COLVIS))
+ return result;
+
+ // check for matching timestamp
+ if (timestamp < other.timestamp)
+ result = 1;
+ else if (timestamp > other.timestamp)
+ result = -1;
+ else
+ result = 0;
+
+ if (result != 0 || part.equals(PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME))
+ return result;
+
+ // check for matching deleted flag
+ if (deleted)
+ result = other.deleted ? 0 : -1;
+ else
+ result = other.deleted ? 1 : 0;
+
+ return result;
+ }
+
+ /**
+ * Compare all elements of a key. The elements (row, column family, column qualifier, column visibility, timestamp, and delete marker) are compared in order
+ * until an unequal element is found. If the row is equal, then compare the column family, etc. The row, column family, column qualifier, and column
+ * visibility are compared lexographically and sorted ascending. The timestamps are compared numerically and sorted descending so that the most recent data
+ * comes first. Lastly, a delete marker of true sorts before a delete marker of false.
+ */
+
++ @Override
+ public int compareTo(Key other) {
+ return compareTo(other, PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME_DEL);
+ }
+
+ @Override
+ public int hashCode() {
+ return WritableComparator.hashBytes(row, row.length) + WritableComparator.hashBytes(colFamily, colFamily.length)
+ + WritableComparator.hashBytes(colQualifier, colQualifier.length) + WritableComparator.hashBytes(colVisibility, colVisibility.length)
+ + (int) (timestamp ^ (timestamp >>> 32));
+ }
+
+ public static String toPrintableString(byte ba[], int offset, int len, int maxLen) {
+ return appendPrintableString(ba, offset, len, maxLen, new StringBuilder()).toString();
+ }
+
+ public static StringBuilder appendPrintableString(byte ba[], int offset, int len, int maxLen, StringBuilder sb) {
+ int plen = Math.min(len, maxLen);
+
+ for (int i = 0; i < plen; i++) {
+ int c = 0xff & ba[offset + i];
+ if (c >= 32 && c <= 126)
+ sb.append((char) c);
+ else
+ sb.append("%" + String.format("%02x;", c));
+ }
+
+ if (len > maxLen) {
+ sb.append("... TRUNCATED");
+ }
+
+ return sb;
+ }
+
+ private StringBuilder rowColumnStringBuilder() {
+ StringBuilder sb = new StringBuilder();
+ appendPrintableString(row, 0, row.length, Constants.MAX_DATA_TO_PRINT, sb);
+ sb.append(" ");
+ appendPrintableString(colFamily, 0, colFamily.length, Constants.MAX_DATA_TO_PRINT, sb);
+ sb.append(":");
+ appendPrintableString(colQualifier, 0, colQualifier.length, Constants.MAX_DATA_TO_PRINT, sb);
+ sb.append(" [");
+ appendPrintableString(colVisibility, 0, colVisibility.length, Constants.MAX_DATA_TO_PRINT, sb);
+ sb.append("]");
+ return sb;
+ }
+
++ @Override
+ public String toString() {
+ StringBuilder sb = rowColumnStringBuilder();
+ sb.append(" ");
+ sb.append(Long.toString(timestamp));
+ sb.append(" ");
+ sb.append(deleted);
+ return sb.toString();
+ }
+
+ public String toStringNoTime() {
+ return rowColumnStringBuilder().toString();
+ }
+
+ /**
+ * Returns the sums of the lengths of the row, column family, column qualifier, and visibility.
+ *
+ * @return row.length + colFamily.length + colQualifier.length + colVisibility.length;
+ */
+ public int getLength() {
+ return row.length + colFamily.length + colQualifier.length + colVisibility.length;
+ }
+
+ /**
+ * Same as {@link #getLength()}.
+ */
+ public int getSize() {
+ return getLength();
+ }
+
+ private static boolean isEqual(byte a1[], byte a2[]) {
+ if (a1 == a2)
+ return true;
+
+ int last = a1.length;
+
+ if (last != a2.length)
+ return false;
+
+ if (last == 0)
+ return true;
+
+ // since sorted data is usually compared in accumulo,
+ // the prefixes will normally be the same... so compare
+ // the last two charachters first.. the most likely place
+ // to have disorder is at end of the strings when the
+ // data is sorted... if those are the same compare the rest
+ // of the data forward... comparing backwards is slower
+ // (compiler and cpu optimized for reading data forward)..
+ // do not want slower comparisons when data is equal...
+ // sorting brings equals data together
+
+ last--;
+
+ if (a1[last] == a2[last]) {
+ for (int i = 0; i < last; i++)
+ if (a1[i] != a2[i])
+ return false;
+ } else {
+ return false;
+ }
+
+ return true;
+
+ }
+
+ /**
+ * Use this to compress a list of keys before sending them via thrift.
+ *
+ * @param param
+ * a list of key/value pairs
+ */
+ public static List<TKeyValue> compress(List<? extends KeyValue> param) {
+
+ List<TKeyValue> tkvl = Arrays.asList(new TKeyValue[param.size()]);
+
+ if (param.size() > 0)
+ tkvl.set(0, new TKeyValue(param.get(0).key.toThrift(), ByteBuffer.wrap(param.get(0).value)));
+
+ for (int i = param.size() - 1; i > 0; i--) {
+ Key prevKey = param.get(i - 1).key;
+ KeyValue kv = param.get(i);
+ Key key = kv.key;
+
+ TKey newKey = null;
+
+ if (isEqual(prevKey.row, key.row)) {
+ newKey = key.toThrift();
+ newKey.row = null;
+ }
+
+ if (isEqual(prevKey.colFamily, key.colFamily)) {
+ if (newKey == null)
+ newKey = key.toThrift();
+ newKey.colFamily = null;
+ }
+
+ if (isEqual(prevKey.colQualifier, key.colQualifier)) {
+ if (newKey == null)
+ newKey = key.toThrift();
+ newKey.colQualifier = null;
+ }
+
+ if (isEqual(prevKey.colVisibility, key.colVisibility)) {
+ if (newKey == null)
+ newKey = key.toThrift();
+ newKey.colVisibility = null;
+ }
+
+ if (newKey == null)
+ newKey = key.toThrift();
+
+ tkvl.set(i, new TKeyValue(newKey, ByteBuffer.wrap(kv.value)));
+ }
+
+ return tkvl;
+ }
+
+ /**
+ * Use this to decompress a list of keys received from thrift.
- *
- * @param param
+ */
-
+ public static void decompress(List<TKeyValue> param) {
+ for (int i = 1; i < param.size(); i++) {
+ TKey prevKey = param.get(i - 1).key;
+ TKey key = param.get(i).key;
+
+ if (key.row == null) {
+ key.row = prevKey.row;
+ }
+ if (key.colFamily == null) {
+ key.colFamily = prevKey.colFamily;
+ }
+ if (key.colQualifier == null) {
+ key.colQualifier = prevKey.colQualifier;
+ }
+ if (key.colVisibility == null) {
+ key.colVisibility = prevKey.colVisibility;
+ }
+ }
+ }
+
+ byte[] getRowBytes() {
+ return row;
+ }
+
+ byte[] getColFamily() {
+ return colFamily;
+ }
+
+ byte[] getColQualifier() {
+ return colQualifier;
+ }
+
+ byte[] getColVisibility() {
+ return colVisibility;
+ }
+
+ public TKey toThrift() {
+ return new TKey(ByteBuffer.wrap(row), ByteBuffer.wrap(colFamily), ByteBuffer.wrap(colQualifier), ByteBuffer.wrap(colVisibility), timestamp);
+ }
+
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ Key r = (Key) super.clone();
+ r.row = Arrays.copyOf(row, row.length);
+ r.colFamily = Arrays.copyOf(colFamily, colFamily.length);
+ r.colQualifier = Arrays.copyOf(colQualifier, colQualifier.length);
+ r.colVisibility = Arrays.copyOf(colVisibility, colVisibility.length);
+ return r;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
index e48914d,0000000..63c594c
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
@@@ -1,783 -1,0 +1,783 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
+/**
+ * keeps track of information needed to identify a tablet
+ * apparently, we only need the endKey and not the start as well
+ *
+ */
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.lang.ref.WeakReference;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.UUID;
+import java.util.WeakHashMap;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.thrift.TKeyExtent;
+import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.hadoop.io.BinaryComparable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+
+public class KeyExtent implements WritableComparable<KeyExtent> {
+
+ private static final WeakHashMap<Text,WeakReference<Text>> tableIds = new WeakHashMap<Text,WeakReference<Text>>();
+
+ private static Text dedupeTableId(Text tableId) {
+ synchronized (tableIds) {
+ WeakReference<Text> etir = tableIds.get(tableId);
+ if (etir != null) {
+ Text eti = etir.get();
+ if (eti != null) {
+ return eti;
+ }
+ }
+
+ tableId = new Text(tableId);
+ tableIds.put(tableId, new WeakReference<Text>(tableId));
+ return tableId;
+ }
+ }
+
+ private Text textTableId;
+ private Text textEndRow;
+ private Text textPrevEndRow;
+
+ private void check() {
+
+ if (getTableId() == null)
+ throw new IllegalArgumentException("null table id not allowed");
+
+ if (getEndRow() == null || getPrevEndRow() == null)
+ return;
+
+ if (getPrevEndRow().compareTo(getEndRow()) >= 0) {
+ throw new IllegalArgumentException("prevEndRow (" + getPrevEndRow() + ") >= endRow (" + getEndRow() + ")");
+ }
+ }
+
+ /**
+ * Default constructor
+ *
+ */
+ public KeyExtent() {
+ this.setTableId(new Text());
+ this.setEndRow(new Text(), false, false);
+ this.setPrevEndRow(new Text(), false, false);
+ }
+
+ public KeyExtent(Text table, Text endRow, Text prevEndRow) {
+ this.setTableId(table);
+ this.setEndRow(endRow, false, true);
+ this.setPrevEndRow(prevEndRow, false, true);
+
+ check();
+ }
+
+ public KeyExtent(KeyExtent extent) {
+ // extent has already deduped table id, so there is no need to do it again
+ this.textTableId = extent.textTableId;
+ this.setEndRow(extent.getEndRow(), false, true);
+ this.setPrevEndRow(extent.getPrevEndRow(), false, true);
+
+ check();
+ }
+
+ public KeyExtent(TKeyExtent tke) {
+ this.setTableId(new Text(ByteBufferUtil.toBytes(tke.table)));
+ this.setEndRow(tke.endRow == null ? null : new Text(ByteBufferUtil.toBytes(tke.endRow)), false, false);
+ this.setPrevEndRow(tke.prevEndRow == null ? null : new Text(ByteBufferUtil.toBytes(tke.prevEndRow)), false, false);
+
+ check();
+ }
+
+ /**
+ * Returns a String representing this extent's entry in the Metadata table
+ *
+ */
+ public Text getMetadataEntry() {
+ return getMetadataEntry(getTableId(), getEndRow());
+ }
+
+ public static Text getMetadataEntry(Text table, Text row) {
+ Text entry = new Text(table);
+
+ if (row == null) {
+ entry.append(new byte[] {'<'}, 0, 1);
+ } else {
+ entry.append(new byte[] {';'}, 0, 1);
+ entry.append(row.getBytes(), 0, row.getLength());
+ }
+
+ return entry;
+
+ }
+
+ // constructor for loading extents from metadata rows
+ public KeyExtent(Text flattenedExtent, Value prevEndRow) {
+ decodeMetadataRow(flattenedExtent);
+
+ // decode the prev row
+ this.setPrevEndRow(decodePrevEndRow(prevEndRow), false, true);
+
+ check();
+ }
+
+ // recreates an encoded extent from a string representation
+ // this encoding is what is stored as the row id of the metadata table
+ public KeyExtent(Text flattenedExtent, Text prevEndRow) {
+
+ decodeMetadataRow(flattenedExtent);
+
+ this.setPrevEndRow(null, false, false);
+ if (prevEndRow != null)
+ this.setPrevEndRow(prevEndRow, false, true);
+
+ check();
+ }
+
+ /**
+ * Sets the extents table id
+ *
+ */
+ public void setTableId(Text tId) {
+
+ if (tId == null)
+ throw new IllegalArgumentException("null table name not allowed");
+
+ this.textTableId = dedupeTableId(tId);
+
+ hashCode = 0;
+ }
+
+ /**
+ * Returns the extent's table id
+ *
+ */
+ public Text getTableId() {
+ return textTableId;
+ }
+
+ private void setEndRow(Text endRow, boolean check, boolean copy) {
+ if (endRow != null)
+ if (copy)
+ this.textEndRow = new Text(endRow);
+ else
+ this.textEndRow = endRow;
+ else
+ this.textEndRow = null;
+
+ hashCode = 0;
+ if (check)
+ check();
+ }
+
+ /**
+ * Sets this extent's end row
+ *
+ */
+ public void setEndRow(Text endRow) {
+ setEndRow(endRow, true, true);
+ }
+
+ /**
+ * Returns this extent's end row
+ *
+ */
+ public Text getEndRow() {
+ return textEndRow;
+ }
+
+ /**
+ * Return the previous extent's end row
+ *
+ */
+ public Text getPrevEndRow() {
+ return textPrevEndRow;
+ }
+
+ private void setPrevEndRow(Text prevEndRow, boolean check, boolean copy) {
+ if (prevEndRow != null)
+ if (copy)
+ this.textPrevEndRow = new Text(prevEndRow);
+ else
+ this.textPrevEndRow = prevEndRow;
+ else
+ this.textPrevEndRow = null;
+
+ hashCode = 0;
+ if (check)
+ check();
+ }
+
+ /**
+ * Sets the previous extent's end row
+ *
+ */
+ public void setPrevEndRow(Text prevEndRow) {
+ setPrevEndRow(prevEndRow, true, true);
+ }
+
+ /**
+ * Populates the extents data fields from a DataInput object
+ *
+ */
++ @Override
+ public void readFields(DataInput in) throws IOException {
+ Text tid = new Text();
+ tid.readFields(in);
+ setTableId(tid);
+ boolean hasRow = in.readBoolean();
+ if (hasRow) {
+ Text er = new Text();
+ er.readFields(in);
+ setEndRow(er, false, false);
+ } else {
+ setEndRow(null, false, false);
+ }
+ boolean hasPrevRow = in.readBoolean();
+ if (hasPrevRow) {
+ Text per = new Text();
+ per.readFields(in);
+ setPrevEndRow(per, false, true);
+ } else {
+ setPrevEndRow((Text) null);
+ }
+
+ hashCode = 0;
+ check();
+ }
+
+ /**
+ * Writes this extent's data fields to a DataOutput object
+ *
+ */
++ @Override
+ public void write(DataOutput out) throws IOException {
+ getTableId().write(out);
+ if (getEndRow() != null) {
+ out.writeBoolean(true);
+ getEndRow().write(out);
+ } else {
+ out.writeBoolean(false);
+ }
+ if (getPrevEndRow() != null) {
+ out.writeBoolean(true);
+ getPrevEndRow().write(out);
+ } else {
+ out.writeBoolean(false);
+ }
+ }
+
+ /**
+ * Returns a String representing the previous extent's entry in the Metadata table
+ *
+ */
+ public Mutation getPrevRowUpdateMutation() {
+ return getPrevRowUpdateMutation(this);
+ }
+
+ /**
+ * Empty start or end rows tell the method there are no start or end rows, and to use all the keyextents that are before the end row if no start row etc.
+ *
+ * @return all the key extents that the rows cover
+ */
+
+ public static Collection<KeyExtent> getKeyExtentsForRange(Text startRow, Text endRow, Set<KeyExtent> kes) {
+ if (kes == null)
+ return Collections.emptyList();
+ if (startRow == null)
+ startRow = new Text();
+ if (endRow == null)
+ endRow = new Text();
+ Collection<KeyExtent> keys = new ArrayList<KeyExtent>();
+ for (KeyExtent ckes : kes) {
+ if (ckes.getPrevEndRow() == null) {
+ if (ckes.getEndRow() == null) {
+ // only tablet
+ keys.add(ckes);
+ } else {
+ // first tablet
+ // if start row = '' then we want everything up to the endRow which will always include the first tablet
+ if (startRow.getLength() == 0) {
+ keys.add(ckes);
+ } else if (ckes.getEndRow().compareTo(startRow) >= 0) {
+ keys.add(ckes);
+ }
+ }
+ } else {
+ if (ckes.getEndRow() == null) {
+ // last tablet
+ // if endRow = '' and we're at the last tablet, add it
+ if (endRow.getLength() == 0) {
+ keys.add(ckes);
+ }
+ if (ckes.getPrevEndRow().compareTo(endRow) < 0) {
+ keys.add(ckes);
+ }
+ } else {
+ // tablet in the middle
+ if (startRow.getLength() == 0) {
+ // no start row
+
+ if (endRow.getLength() == 0) {
+ // no start & end row
+ keys.add(ckes);
+ } else {
+ // just no start row
+ if (ckes.getPrevEndRow().compareTo(endRow) < 0) {
+ keys.add(ckes);
+ }
+ }
+ } else if (endRow.getLength() == 0) {
+ // no end row
+ if (ckes.getEndRow().compareTo(startRow) >= 0) {
+ keys.add(ckes);
+ }
+ } else {
+ // no null prevend or endrows and no empty string start or end rows
+ if ((ckes.getPrevEndRow().compareTo(endRow) < 0 && ckes.getEndRow().compareTo(startRow) >= 0)) {
+ keys.add(ckes);
+ }
+ }
+
+ }
+ }
+ }
+ return keys;
+ }
+
+ public static Text decodePrevEndRow(Value ibw) {
+ Text per = null;
+
+ if (ibw.get()[0] != 0) {
+ per = new Text();
+ per.set(ibw.get(), 1, ibw.get().length - 1);
+ }
+
+ return per;
+ }
+
+ public static Value encodePrevEndRow(Text per) {
+ if (per == null)
+ return new Value(new byte[] {0});
+ byte[] b = new byte[per.getLength() + 1];
+ b[0] = 1;
+ System.arraycopy(per.getBytes(), 0, b, 1, per.getLength());
+ return new Value(b);
+ }
+
+ public static Mutation getPrevRowUpdateMutation(KeyExtent ke) {
+ Mutation m = new Mutation(ke.getMetadataEntry());
+ Constants.METADATA_PREV_ROW_COLUMN.put(m, encodePrevEndRow(ke.getPrevEndRow()));
+ return m;
+ }
+
+ /**
+ * Compares extents based on rows
+ *
+ */
++ @Override
+ public int compareTo(KeyExtent other) {
+
+ int result = getTableId().compareTo(other.getTableId());
+ if (result != 0)
+ return result;
+
+ if (this.getEndRow() == null) {
+ if (other.getEndRow() != null)
+ return 1;
+ } else {
+ if (other.getEndRow() == null)
+ return -1;
+
+ result = getEndRow().compareTo(other.getEndRow());
+ if (result != 0)
+ return result;
+ }
+ if (this.getPrevEndRow() == null) {
+ if (other.getPrevEndRow() == null)
+ return 0;
+ return -1;
+ }
+ if (other.getPrevEndRow() == null)
+ return 1;
+ return this.getPrevEndRow().compareTo(other.getPrevEndRow());
+ }
+
+ private int hashCode = 0;
+
+ @Override
+ public int hashCode() {
+ if (hashCode != 0)
+ return hashCode;
+
+ int prevEndRowHash = 0;
+ int endRowHash = 0;
+ if (this.getEndRow() != null) {
+ endRowHash = this.getEndRow().hashCode();
+ }
+
+ if (this.getPrevEndRow() != null) {
+ prevEndRowHash = this.getPrevEndRow().hashCode();
+ }
+
+ hashCode = getTableId().hashCode() + endRowHash + prevEndRowHash;
+ return hashCode;
+ }
+
+ private boolean equals(Text t1, Text t2) {
+ if (t1 == null || t2 == null)
+ return t1 == t2;
+
+ return t1.equals(t2);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this)
+ return true;
+ if (!(o instanceof KeyExtent))
+ return false;
+ KeyExtent oke = (KeyExtent) o;
+ return textTableId.equals(oke.textTableId) && equals(textEndRow, oke.textEndRow) && equals(textPrevEndRow, oke.textPrevEndRow);
+ }
+
+ @Override
+ public String toString() {
+ String endRowString;
+ String prevEndRowString;
+ String tableIdString = getTableId().toString().replaceAll(";", "\\\\;").replaceAll("\\\\", "\\\\\\\\");
+
+ if (getEndRow() == null)
+ endRowString = "<";
+ else
+ endRowString = ";" + TextUtil.truncate(getEndRow()).toString().replaceAll(";", "\\\\;").replaceAll("\\\\", "\\\\\\\\");
+
+ if (getPrevEndRow() == null)
+ prevEndRowString = "<";
+ else
+ prevEndRowString = ";" + TextUtil.truncate(getPrevEndRow()).toString().replaceAll(";", "\\\\;").replaceAll("\\\\", "\\\\\\\\");
+
+ return tableIdString + endRowString + prevEndRowString;
+ }
+
+ public UUID getUUID() {
+ try {
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+
+ // to get a unique hash it is important to encode the data
+ // like it is being serialized
+
+ this.write(dos);
+
+ dos.close();
+
+ return UUID.nameUUIDFromBytes(baos.toByteArray());
+
+ } catch (IOException e) {
+ // should not happen since we are writing to memory
+ throw new RuntimeException(e);
+ }
+ }
+
+ // note: this is only the encoding of the table id and the last row, not the prev row
+ /**
+ * Populates the extent's fields based on a flatted extent
+ *
+ */
+ private void decodeMetadataRow(Text flattenedExtent) {
+ int semiPos = -1;
+ int ltPos = -1;
+
+ for (int i = 0; i < flattenedExtent.getLength(); i++) {
+ if (flattenedExtent.getBytes()[i] == ';' && semiPos < 0) {
+ // want the position of the first semicolon
+ semiPos = i;
+ }
+
+ if (flattenedExtent.getBytes()[i] == '<') {
+ ltPos = i;
+ }
+ }
+
+ if (semiPos < 0 && ltPos < 0) {
+ throw new IllegalArgumentException("Metadata row does not contain ; or < " + flattenedExtent);
+ }
+
+ if (semiPos < 0) {
+
+ if (ltPos != flattenedExtent.getLength() - 1) {
+ throw new IllegalArgumentException("< must come at end of Metadata row " + flattenedExtent);
+ }
+
+ Text tableId = new Text();
+ tableId.set(flattenedExtent.getBytes(), 0, flattenedExtent.getLength() - 1);
+ this.setTableId(tableId);
+ this.setEndRow(null, false, false);
+ } else {
+
+ Text tableId = new Text();
+ tableId.set(flattenedExtent.getBytes(), 0, semiPos);
+
+ Text endRow = new Text();
+ endRow.set(flattenedExtent.getBytes(), semiPos + 1, flattenedExtent.getLength() - (semiPos + 1));
+
+ this.setTableId(tableId);
+
+ this.setEndRow(endRow, false, false);
+ }
+ }
+
+ public static byte[] tableOfMetadataRow(Text row) {
+ KeyExtent ke = new KeyExtent();
+ ke.decodeMetadataRow(row);
+ return TextUtil.getBytes(ke.getTableId());
+ }
+
+ public boolean contains(final ByteSequence bsrow) {
+ if (bsrow == null) {
+ throw new IllegalArgumentException("Passing null to contains is ambiguous, could be in first or last extent of table");
+ }
+
+ BinaryComparable row = new BinaryComparable() {
+
+ @Override
+ public int getLength() {
+ return bsrow.length();
+ }
+
+ @Override
+ public byte[] getBytes() {
+ if (bsrow.isBackedByArray() && bsrow.offset() == 0)
+ return bsrow.getBackingArray();
+
+ return bsrow.toArray();
+ }
+ };
+
+ if ((this.getPrevEndRow() == null || this.getPrevEndRow().compareTo(row) < 0) && (this.getEndRow() == null || this.getEndRow().compareTo(row) >= 0)) {
+ return true;
+ }
+ return false;
+ }
+
+ public boolean contains(BinaryComparable row) {
+ if (row == null) {
+ throw new IllegalArgumentException("Passing null to contains is ambiguous, could be in first or last extent of table");
+ }
+
+ if ((this.getPrevEndRow() == null || this.getPrevEndRow().compareTo(row) < 0) && (this.getEndRow() == null || this.getEndRow().compareTo(row) >= 0)) {
+ return true;
+ }
+ return false;
+ }
+
+ public Range toDataRange() {
+ return new Range(getPrevEndRow(), false, getEndRow(), true);
+ }
+
+ public Range toMetadataRange() {
+ Text metadataPrevRow = new Text(getTableId());
+ metadataPrevRow.append(new byte[] {';'}, 0, 1);
+ if (getPrevEndRow() != null) {
+ metadataPrevRow.append(getPrevEndRow().getBytes(), 0, getPrevEndRow().getLength());
+ }
+
+ Range range = new Range(metadataPrevRow, getPrevEndRow() == null, getMetadataEntry(), true);
+ return range;
+ }
+
+ public static SortedSet<KeyExtent> findChildren(KeyExtent ke, SortedSet<KeyExtent> tablets) {
+
+ SortedSet<KeyExtent> children = null;
+
+ for (KeyExtent tabletKe : tablets) {
+
+ if (ke.getPrevEndRow() == tabletKe.getPrevEndRow() || ke.getPrevEndRow() != null && tabletKe.getPrevEndRow() != null
+ && tabletKe.getPrevEndRow().compareTo(ke.getPrevEndRow()) == 0) {
+ children = new TreeSet<KeyExtent>();
+ }
+
+ if (children != null) {
+ children.add(tabletKe);
+ }
+
+ if (ke.getEndRow() == tabletKe.getEndRow() || ke.getEndRow() != null && tabletKe.getEndRow() != null
+ && tabletKe.getEndRow().compareTo(ke.getEndRow()) == 0) {
+ return children;
+ }
+ }
+
+ return new TreeSet<KeyExtent>();
+ }
+
+ public static KeyExtent findContainingExtent(KeyExtent extent, SortedSet<KeyExtent> extents) {
+
+ KeyExtent lookupExtent = new KeyExtent(extent);
+ lookupExtent.setPrevEndRow((Text) null);
+
+ SortedSet<KeyExtent> tailSet = extents.tailSet(lookupExtent);
+
+ if (tailSet.isEmpty()) {
+ return null;
+ }
+
+ KeyExtent first = tailSet.first();
+
+ if (first.getTableId().compareTo(extent.getTableId()) != 0) {
+ return null;
+ }
+
+ if (first.getPrevEndRow() == null) {
+ return first;
+ }
+
+ if (extent.getPrevEndRow() == null) {
+ return null;
+ }
+
+ if (extent.getPrevEndRow().compareTo(first.getPrevEndRow()) >= 0)
+ return first;
+ return null;
+ }
+
+ private static boolean startsAfter(KeyExtent nke, KeyExtent ke) {
+
+ int tiCmp = ke.getTableId().compareTo(nke.getTableId());
+
+ if (tiCmp > 0) {
+ return true;
+ }
+
+ return ke.getPrevEndRow() != null && nke.getEndRow() != null && ke.getPrevEndRow().compareTo(nke.getEndRow()) >= 0;
+ }
+
+ private static Text rowAfterPrevRow(KeyExtent nke) {
+ Text row = new Text(nke.getPrevEndRow());
+ row.append(new byte[] {0}, 0, 1);
+ return row;
+ }
+
+ // Some duplication with TabletLocatorImpl
+ public static Set<KeyExtent> findOverlapping(KeyExtent nke, SortedSet<KeyExtent> extents) {
+ if (nke == null || extents == null || extents.isEmpty())
+ return Collections.emptySet();
+
+ SortedSet<KeyExtent> start;
+
+ if (nke.getPrevEndRow() != null) {
+ Text row = rowAfterPrevRow(nke);
+ KeyExtent lookupKey = new KeyExtent(nke.getTableId(), row, null);
+ start = extents.tailSet(lookupKey);
+ } else {
+ KeyExtent lookupKey = new KeyExtent(nke.getTableId(), new Text(), null);
+ start = extents.tailSet(lookupKey);
+ }
+
+ TreeSet<KeyExtent> result = new TreeSet<KeyExtent>();
+ for (KeyExtent ke : start) {
+ if (startsAfter(nke, ke)) {
+ break;
+ }
+ result.add(ke);
+ }
+ return result;
+ }
+
+ public boolean overlaps(KeyExtent other) {
+ SortedSet<KeyExtent> set = new TreeSet<KeyExtent>();
+ set.add(other);
+ return !findOverlapping(this, set).isEmpty();
+ }
+
+ // Specialization of findOverlapping(KeyExtent, SortedSet<KeyExtent> to work with SortedMap
+ public static Set<KeyExtent> findOverlapping(KeyExtent nke, SortedMap<KeyExtent,? extends Object> extents) {
+ if (nke == null || extents == null || extents.isEmpty())
+ return Collections.emptySet();
+
+ SortedMap<KeyExtent,? extends Object> start;
+
+ if (nke.getPrevEndRow() != null) {
+ Text row = rowAfterPrevRow(nke);
+ KeyExtent lookupKey = new KeyExtent(nke.getTableId(), row, null);
+ start = extents.tailMap(lookupKey);
+ } else {
+ KeyExtent lookupKey = new KeyExtent(nke.getTableId(), new Text(), null);
+ start = extents.tailMap(lookupKey);
+ }
+
+ TreeSet<KeyExtent> result = new TreeSet<KeyExtent>();
+ for (Entry<KeyExtent,? extends Object> entry : start.entrySet()) {
+ KeyExtent ke = entry.getKey();
+ if (startsAfter(nke, ke)) {
+ break;
+ }
+ result.add(ke);
+ }
+ return result;
+ }
+
+ public static Text getMetadataEntry(KeyExtent extent) {
+ return getMetadataEntry(extent.getTableId(), extent.getEndRow());
+ }
+
+ public TKeyExtent toThrift() {
+ return new TKeyExtent(TextUtil.getByteBuffer(textTableId), textEndRow == null ? null : TextUtil.getByteBuffer(textEndRow), textPrevEndRow == null ? null
+ : TextUtil.getByteBuffer(textPrevEndRow));
+ }
+
- /**
- * @param prevExtent
- */
+ public boolean isPreviousExtent(KeyExtent prevExtent) {
+ if (prevExtent == null)
+ return getPrevEndRow() == null;
+
+ if (!prevExtent.getTableId().equals(getTableId()))
+ throw new IllegalArgumentException("Cannot compare accross tables " + prevExtent + " " + this);
+
+ if (prevExtent.getEndRow() == null)
+ return false;
+
+ if (getPrevEndRow() == null)
+ return false;
+
+ return prevExtent.getEndRow().equals(getPrevEndRow());
+ }
+
+ public boolean isMeta() {
+ return getTableId().toString().equals(Constants.METADATA_TABLE_ID);
+ }
+
+ public boolean isRootTablet() {
+ return this.compareTo(Constants.ROOT_TABLET_EXTENT) == 0;
+ }
+}
[19/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
index efadfae,0000000..09fbbd2
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
@@@ -1,279 -1,0 +1,278 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
+import java.net.Socket;
+import java.net.URLEncoder;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.net.SocketFactory;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.commons.io.filefilter.WildcardFileFilter;
+import org.apache.commons.lang.math.LongRange;
+import org.apache.log4j.Category;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.Filter;
+import org.apache.log4j.spi.LocationInfo;
+import org.apache.log4j.spi.LoggingEvent;
+import org.apache.log4j.spi.ThrowableInformation;
+import org.apache.log4j.varia.LevelRangeFilter;
+import org.apache.log4j.xml.XMLLayout;
+
+import com.beust.jcommander.IStringConverter;
+import com.beust.jcommander.Parameter;
+
+public class SendLogToChainsaw extends XMLLayout {
+
+ private static Pattern logPattern = Pattern.compile(
+ "^(\\d\\d)\\s(\\d\\d):(\\d\\d):(\\d\\d),(\\d\\d\\d)\\s\\[(.*)\\]\\s(TRACE|DEBUG|INFO|WARN|FATAL|ERROR)\\s*?:(.*)$", Pattern.UNIX_LINES);
+
+ private File[] logFiles = null;
+
+ private SocketFactory factory = SocketFactory.getDefault();
+
+ private WildcardFileFilter fileFilter = null;
+
+ private Socket socket = null;
+
+ private Pattern lineFilter = null;
+
+ private LongRange dateFilter = null;
+
+ private LevelRangeFilter levelFilter = null;
+
+ public SendLogToChainsaw(String directory, String fileNameFilter, String host, int port, Date start, Date end, String regex, String level) throws Exception {
+
+ // Set up the file name filter
+ if (null != fileNameFilter) {
+ fileFilter = new WildcardFileFilter(fileNameFilter);
+ } else {
+ fileFilter = new WildcardFileFilter("*");
+ }
+
+ // Get the list of files that match
+ File dir = new File(directory);
+ if (dir.isDirectory()) {
+ logFiles = dir.listFiles((FilenameFilter) fileFilter);
+ } else {
+ throw new IllegalArgumentException(directory + " is not a directory or is not readable.");
+ }
+
+ if (logFiles.length == 0) {
+ throw new IllegalArgumentException("No files match the supplied filter.");
+ }
+
+ socket = factory.createSocket(host, port);
+
+ lineFilter = Pattern.compile(regex);
+
+ // Create Date Filter
+ if (null != start) {
+ if (end == null)
+ end = new Date(System.currentTimeMillis());
+ dateFilter = new LongRange(start.getTime(), end.getTime());
+ }
+
+ if (null != level) {
+ Level base = Level.toLevel(level.toUpperCase());
+ levelFilter = new LevelRangeFilter();
+ levelFilter.setAcceptOnMatch(true);
+ levelFilter.setLevelMin(base);
+ levelFilter.setLevelMax(Level.FATAL);
+ }
+ }
+
+ public void processLogFiles() throws IOException {
+ String line = null;
+ String out = null;
+ InputStreamReader isReader = null;
+ BufferedReader reader = null;
+ try {
+ for (File log : logFiles) {
+ // Parse the server type and name from the log file name
+ String threadName = log.getName().substring(0, log.getName().indexOf("."));
+ try {
+ isReader = new InputStreamReader(new FileInputStream(log), Constants.UTF8);
+ } catch (FileNotFoundException e) {
+ System.out.println("Unable to find file: " + log.getAbsolutePath());
+ throw e;
+ }
+ reader = new BufferedReader(isReader);
+
+ try {
+ line = reader.readLine();
+ while (null != line) {
+ out = convertLine(line, threadName);
+ if (null != out) {
+ if (socket != null && socket.isConnected())
+ socket.getOutputStream().write(out.getBytes(Constants.UTF8));
+ else
+ System.err.println("Unable to send data to transport");
+ }
+ line = reader.readLine();
+ }
+ } catch (IOException e) {
+ System.out.println("Error processing line: " + line + ". Output was " + out);
+ throw e;
+ } finally {
+ if (reader != null) {
+ reader.close();
+ }
+ if (isReader != null) {
+ isReader.close();
+ }
+ }
+ }
+ } finally {
+ if (socket != null && socket.isConnected()) {
+ socket.close();
+ }
+ }
+ }
+
+ private String convertLine(String line, String threadName) throws UnsupportedEncodingException {
+ String result = null;
+ Matcher m = logPattern.matcher(line);
+ if (m.matches()) {
+
+ Calendar cal = Calendar.getInstance();
+ cal.setTime(new Date(System.currentTimeMillis()));
+ Integer date = Integer.parseInt(m.group(1));
+ Integer hour = Integer.parseInt(m.group(2));
+ Integer min = Integer.parseInt(m.group(3));
+ Integer sec = Integer.parseInt(m.group(4));
+ Integer ms = Integer.parseInt(m.group(5));
+ String clazz = m.group(6);
+ String level = m.group(7);
+ String message = m.group(8);
+ // Apply the regex filter if supplied
+ if (null != lineFilter) {
+ Matcher match = lineFilter.matcher(message);
+ if (!match.matches())
+ return null;
+ }
+ // URL encode the message
+ message = URLEncoder.encode(message, "UTF-8");
+ // Assume that we are processing logs from today.
+ // If the date in the line is greater than today, then it must be
+ // from the previous month.
+ cal.set(Calendar.DATE, date);
+ cal.set(Calendar.HOUR_OF_DAY, hour);
+ cal.set(Calendar.MINUTE, min);
+ cal.set(Calendar.SECOND, sec);
+ cal.set(Calendar.MILLISECOND, ms);
+ if (date > cal.get(Calendar.DAY_OF_MONTH)) {
+ cal.add(Calendar.MONTH, -1);
+ }
+ long ts = cal.getTimeInMillis();
+ // If this event is not between the start and end dates, then skip it.
+ if (null != dateFilter && !dateFilter.containsLong(ts))
+ return null;
+ Category c = Logger.getLogger(clazz);
+ Level l = Level.toLevel(level);
+ LoggingEvent event = new LoggingEvent(clazz, c, ts, l, message, threadName, (ThrowableInformation) null, (String) null, (LocationInfo) null,
+ (Map<?,?>) null);
+ // Check the log level filter
+ if (null != levelFilter && (levelFilter.decide(event) == Filter.DENY)) {
+ return null;
+ }
+ result = format(event);
+ }
+ return result;
+ }
+
+ private static class DateConverter implements IStringConverter<Date> {
+ @Override
+ public Date convert(String value) {
+ SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
+ try {
+ return formatter.parse(value);
+ } catch (ParseException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+ private static class Opts extends Help {
+
+ @Parameter(names={"-d", "--logDirectory"}, description="ACCUMULO log directory path", required=true)
+ String dir;
+
+ @Parameter(names={"-f", "--fileFilter"}, description="filter to apply to names of logs")
+ String filter;
+
+ @Parameter(names={"-h", "--host"}, description="host where chainsaw is running", required=true)
+ String hostname;
+
+ @Parameter(names={"-p", "--port"}, description="port where XMLSocketReceiver is listening", required=true)
+ int portnum;
+
+ @Parameter(names={"-s", "--start"}, description="start date filter (yyyyMMddHHmmss)", required=true, converter=DateConverter.class)
+ Date startDate;
+
+ @Parameter(names={"-e", "--end"}, description="end date filter (yyyyMMddHHmmss)", required=true, converter=DateConverter.class)
+ Date endDate;
+
+ @Parameter(names={"-l", "--level"}, description="filter log level")
+ String level;
+
+ @Parameter(names={"-m", "--messageFilter"}, description="regex filter for log messages")
+ String regex;
+ }
+
+
+
+
+ /**
+ *
+ * @param args
+ * 0: path to log directory parameter
+ * 1: filter to apply for logs to include (uses wildcards (i.e. logger* and IS case sensitive) parameter
+ * 2: chainsaw host parameter
+ * 3: chainsaw port parameter
+ * 4: start date filter parameter
+ * 5: end date filter parameter
+ * 6: optional regex filter to match on each log4j message parameter
+ * 7: optional level filter
- * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(SendLogToChainsaw.class.getName(), args);
+
+ SendLogToChainsaw c = new SendLogToChainsaw(opts.dir, opts.filter, opts.hostname, opts.portnum, opts.startDate, opts.endDate, opts.regex, opts.level);
+ c.processLogFiles();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/TServerUtils.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/TServerUtils.java
index 50476a2,0000000..fa4de30
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/TServerUtils.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/TServerUtils.java
@@@ -1,313 -1,0 +1,314 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.UnknownHostException;
+import java.nio.channels.ServerSocketChannel;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.LoggingRunnable;
+import org.apache.accumulo.core.util.SimpleThreadPool;
+import org.apache.accumulo.core.util.TBufferedSocket;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.thrift.metrics.ThriftMetrics;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+import org.apache.thrift.TProcessor;
+import org.apache.thrift.TProcessorFactory;
+import org.apache.thrift.protocol.TProtocol;
+import org.apache.thrift.server.TServer;
+import org.apache.thrift.server.TThreadPoolServer;
+import org.apache.thrift.transport.TNonblockingSocket;
+import org.apache.thrift.transport.TServerTransport;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+
+public class TServerUtils {
+ private static final Logger log = Logger.getLogger(TServerUtils.class);
+
+ public static final ThreadLocal<String> clientAddress = new ThreadLocal<String>();
+
+ public static class ServerPort {
+ public final TServer server;
+ public final int port;
+
+ public ServerPort(TServer server, int port) {
+ this.server = server;
+ this.port = port;
+ }
+ }
+
+ /**
+ * Start a server, at the given port, or higher, if that port is not available.
+ *
+ * @param portHintProperty
+ * the port to attempt to open, can be zero, meaning "any available port"
+ * @param processor
+ * the service to be started
+ * @param serverName
+ * the name of the class that is providing the service
+ * @param threadName
+ * name this service's thread for better debugging
- * @param portSearchProperty
- * @param minThreadProperty
- * @param timeBetweenThreadChecksProperty
+ * @return the server object created, and the port actually used
+ * @throws UnknownHostException
+ * when we don't know our own address
+ */
+ public static ServerPort startServer(AccumuloConfiguration conf, Property portHintProperty, TProcessor processor, String serverName, String threadName,
+ Property portSearchProperty,
+ Property minThreadProperty,
+ Property timeBetweenThreadChecksProperty,
+ Property maxMessageSizeProperty) throws UnknownHostException {
+ int portHint = conf.getPort(portHintProperty);
+ int minThreads = 2;
+ if (minThreadProperty != null)
+ minThreads = conf.getCount(minThreadProperty);
+ long timeBetweenThreadChecks = 1000;
+ if (timeBetweenThreadChecksProperty != null)
+ timeBetweenThreadChecks = conf.getTimeInMillis(timeBetweenThreadChecksProperty);
+ long maxMessageSize = 10 * 1000 * 1000;
+ if (maxMessageSizeProperty != null)
+ maxMessageSize = conf.getMemoryInBytes(maxMessageSizeProperty);
+ boolean portSearch = false;
+ if (portSearchProperty != null)
+ portSearch = conf.getBoolean(portSearchProperty);
+ Random random = new Random();
+ for (int j = 0; j < 100; j++) {
+
+ // Are we going to slide around, looking for an open port?
+ int portsToSearch = 1;
+ if (portSearch)
+ portsToSearch = 1000;
+
+ for (int i = 0; i < portsToSearch; i++) {
+ int port = portHint + i;
+ if (portHint != 0 && i > 0)
+ port = 1024 + random.nextInt(65535 - 1024);
+ if (port > 65535)
+ port = 1024 + port % (65535 - 1024);
+ try {
+ return TServerUtils.startTServer(port, processor, serverName, threadName, minThreads, timeBetweenThreadChecks, maxMessageSize);
+ } catch (Exception ex) {
+ log.info("Unable to use port " + port + ", retrying. (Thread Name = " + threadName + ")");
+ UtilWaitThread.sleep(250);
+ }
+ }
+ }
+ throw new UnknownHostException("Unable to find a listen port");
+ }
+
+ public static class TimedProcessor implements TProcessor {
+
+ final TProcessor other;
+ ThriftMetrics metrics = null;
+ long idleStart = 0;
+
+ TimedProcessor(TProcessor next, String serverName, String threadName) {
+ this.other = next;
+ // Register the metrics MBean
+ try {
+ metrics = new ThriftMetrics(serverName, threadName);
+ metrics.register();
+ } catch (Exception e) {
+ log.error("Exception registering MBean with MBean Server", e);
+ }
+ idleStart = System.currentTimeMillis();
+ }
+
+ @Override
+ public boolean process(TProtocol in, TProtocol out) throws TException {
+ long now = 0;
+ if (metrics.isEnabled()) {
+ now = System.currentTimeMillis();
+ metrics.add(ThriftMetrics.idle, (now - idleStart));
+ }
+ try {
+ try {
+ return other.process(in, out);
+ } catch (NullPointerException ex) {
+ // THRIFT-1447 - remove with thrift 0.9
+ return true;
+ }
+ } finally {
+ if (metrics.isEnabled()) {
+ idleStart = System.currentTimeMillis();
+ metrics.add(ThriftMetrics.execute, idleStart - now);
+ }
+ }
+ }
+ }
+
+ public static class ClientInfoProcessorFactory extends TProcessorFactory {
+
+ public ClientInfoProcessorFactory(TProcessor processor) {
+ super(processor);
+ }
+
++ @Override
+ public TProcessor getProcessor(TTransport trans) {
+ if (trans instanceof TBufferedSocket) {
+ TBufferedSocket tsock = (TBufferedSocket) trans;
+ clientAddress.set(tsock.getClientString());
+ }
+ return super.getProcessor(trans);
+ }
+ }
+
+ public static class THsHaServer extends org.apache.thrift.server.THsHaServer {
+ public THsHaServer(Args args) {
+ super(args);
+ }
+
++ @Override
+ protected Runnable getRunnable(FrameBuffer frameBuffer) {
+ return new Invocation(frameBuffer);
+ }
+
+ private class Invocation implements Runnable {
+
+ private final FrameBuffer frameBuffer;
+
+ public Invocation(final FrameBuffer frameBuffer) {
+ this.frameBuffer = frameBuffer;
+ }
+
++ @Override
+ public void run() {
+ if (frameBuffer.trans_ instanceof TNonblockingSocket) {
+ TNonblockingSocket tsock = (TNonblockingSocket) frameBuffer.trans_;
+ Socket sock = tsock.getSocketChannel().socket();
+ clientAddress.set(sock.getInetAddress().getHostAddress() + ":" + sock.getPort());
+ }
+ frameBuffer.invoke();
+ }
+ }
+ }
+
+ public static ServerPort startHsHaServer(int port, TProcessor processor, final String serverName, String threadName, final int numThreads,
+ long timeBetweenThreadChecks, long maxMessageSize) throws TTransportException {
+ TNonblockingServerSocket transport = new TNonblockingServerSocket(port);
+ if (port == 0) {
+ port = transport.getPort();
+ }
+ THsHaServer.Args options = new THsHaServer.Args(transport);
+ options.protocolFactory(ThriftUtil.protocolFactory());
+ options.transportFactory(ThriftUtil.transportFactory(maxMessageSize));
+ options.maxReadBufferBytes = maxMessageSize;
+ options.stopTimeoutVal(5);
+ /*
+ * Create our own very special thread pool.
+ */
+ final ThreadPoolExecutor pool = new SimpleThreadPool(numThreads, "ClientPool");
+ // periodically adjust the number of threads we need by checking how busy our threads are
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ if (pool.getCorePoolSize() <= pool.getActiveCount()) {
+ int larger = pool.getCorePoolSize() + Math.min(pool.getQueue().size(), 2);
+ log.info("Increasing server thread pool size on " + serverName + " to " + larger);
+ pool.setMaximumPoolSize(larger);
+ pool.setCorePoolSize(larger);
+ } else {
+ if (pool.getCorePoolSize() > pool.getActiveCount() + 3) {
+ int smaller = Math.max(numThreads, pool.getCorePoolSize() - 1);
+ if (smaller != pool.getCorePoolSize()) {
+ // there is a race condition here... the active count could be higher by the time
+ // we decrease the core pool size... so the active count could end up higher than
+ // the core pool size, in which case everything will be queued... the increase case
+ // should handle this and prevent deadlock
+ log.info("Decreasing server thread pool size on " + serverName + " to " + smaller);
+ pool.setCorePoolSize(smaller);
+ }
+ }
+ }
+ }
+ }, timeBetweenThreadChecks, timeBetweenThreadChecks);
+ options.executorService(pool);
+ processor = new TServerUtils.TimedProcessor(processor, serverName, threadName);
+ options.processorFactory(new TProcessorFactory(processor));
+ return new ServerPort(new THsHaServer(options), port);
+ }
+
+ public static ServerPort startThreadPoolServer(int port, TProcessor processor, String serverName, String threadName, int numThreads)
+ throws TTransportException {
+
+ // if port is zero, then we must bind to get the port number
+ ServerSocket sock;
+ try {
+ sock = ServerSocketChannel.open().socket();
+ sock.setReuseAddress(true);
+ sock.bind(new InetSocketAddress(port));
+ port = sock.getLocalPort();
+ } catch (IOException ex) {
+ throw new TTransportException(ex);
+ }
+ TServerTransport transport = new TBufferedServerSocket(sock, 32 * 1024);
+ TThreadPoolServer.Args options = new TThreadPoolServer.Args(transport);
+ options.protocolFactory(ThriftUtil.protocolFactory());
+ options.transportFactory(ThriftUtil.transportFactory());
+ processor = new TServerUtils.TimedProcessor(processor, serverName, threadName);
+ options.processorFactory(new ClientInfoProcessorFactory(processor));
+ return new ServerPort(new TThreadPoolServer(options), port);
+ }
+
+ public static ServerPort startTServer(int port, TProcessor processor, String serverName, String threadName, int numThreads, long timeBetweenThreadChecks, long maxMessageSize)
+ throws TTransportException {
+ ServerPort result = startHsHaServer(port, processor, serverName, threadName, numThreads, timeBetweenThreadChecks, maxMessageSize);
+ // ServerPort result = startThreadPoolServer(port, processor, serverName, threadName, -1);
+ final TServer finalServer = result.server;
+ Runnable serveTask = new Runnable() {
++ @Override
+ public void run() {
+ try {
+ finalServer.serve();
+ } catch (Error e) {
+ Halt.halt("Unexpected error in TThreadPoolServer " + e + ", halting.");
+ }
+ }
+ };
+ serveTask = new LoggingRunnable(TServerUtils.log, serveTask);
+ Thread thread = new Daemon(serveTask, threadName);
+ thread.start();
+ return result;
+ }
+
+ // Existing connections will keep our thread running: reach in with reflection and insist that they shutdown.
+ public static void stopTServer(TServer s) {
+ if (s == null)
+ return;
+ s.stop();
+ try {
+ Field f = s.getClass().getDeclaredField("executorService_");
+ f.setAccessible(true);
+ ExecutorService es = (ExecutorService) f.get(s);
+ es.shutdownNow();
+ } catch (Exception e) {
+ TServerUtils.log.error("Unable to call shutdownNow", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
index c3f4a72,0000000..92e0674
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
@@@ -1,48 -1,0 +1,45 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+
+import com.beust.jcommander.Parameter;
+
+public class TableDiskUsage {
+
+ static class Opts extends ClientOpts {
+ @Parameter(description=" <table> { <table> ... } ")
+ List<String> tables = new ArrayList<String>();
+ }
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ FileSystem fs = FileSystem.get(new Configuration());
+ Opts opts = new Opts();
+ opts.parseArgs(TableDiskUsage.class.getName(), args);
+ Connector conn = opts.getConnector();
+ org.apache.accumulo.core.util.TableDiskUsage.printDiskUsage(DefaultConfiguration.getInstance(), opts.tables, fs, conn, false);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
index 2fc0bd3,0000000..34c2151
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
@@@ -1,75 -1,0 +1,72 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.util.List;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+
+import com.beust.jcommander.Parameter;
+
+public class TabletServerLocks {
+
+ static class Opts extends Help {
+ @Parameter(names="-list")
+ boolean list = false;
+ @Parameter(names="-delete")
+ String delete = null;
+ }
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+
+ Instance instance = HdfsZooInstance.getInstance();
+ String tserverPath = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
+ Opts opts = new Opts();
+ opts.parseArgs(TabletServerLocks.class.getName(), args);
+
+ ZooCache cache = new ZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
+
+ if (opts.list) {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ List<String> tabletServers = zoo.getChildren(tserverPath);
+
+ for (String tabletServer : tabletServers) {
+ byte[] lockData = ZooLock.getLockData(cache, tserverPath + "/" + tabletServer, null);
+ String holder = null;
+ if (lockData != null) {
+ holder = new String(lockData, Constants.UTF8);
+ }
+
+ System.out.printf("%32s %16s%n", tabletServer, holder);
+ }
+ } else if (opts.delete != null) {
+ ZooLock.deleteLock(tserverPath + "/" + args[1]);
+ } else {
+ System.out.println("Usage : " + TabletServerLocks.class.getName() + " -list|-delete <tserver lock>");
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
----------------------------------------------------------------------
diff --cc start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
index 5f7fd5e,0000000..1e2b4b5
mode 100644,000000..100644
--- a/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
+++ b/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
@@@ -1,255 -1,0 +1,252 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.start.classloader;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import org.apache.log4j.Logger;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+/**
+ *
+ */
+public class AccumuloClassLoader {
+
+ public static final String CLASSPATH_PROPERTY_NAME = "general.classpaths";
+
+ public static final String ACCUMULO_CLASSPATH_VALUE =
+ "$ACCUMULO_CONF_DIR,\n" +
+ "$ACCUMULO_HOME/lib/[^.].*.jar,\n" +
+ "$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n" +
+ "$HADOOP_CONF_DIR,\n" +
+ "$HADOOP_PREFIX/[^.].*.jar,\n" +
+ "$HADOOP_PREFIX/lib/[^.].*.jar,\n" +
+ "$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n" +
+ "$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n" +
+ "$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n" +
+ "$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n" +
+ "/usr/lib/hadoop/[^.].*.jar,\n" +
+ "/usr/lib/hadoop/lib/[^.].*.jar,\n" +
+ "/usr/lib/hadoop-hdfs/[^.].*.jar,\n" +
+ "/usr/lib/hadoop-mapreduce/[^.].*.jar,\n" +
+ "/usr/lib/hadoop-yarn/[^.].*.jar,\n"
+ ;
+
+ private static String SITE_CONF;
+
+ private static URLClassLoader classloader;
+
+ private static Logger log = Logger.getLogger(AccumuloClassLoader.class);
+
+ static {
+ String configFile = System.getProperty("org.apache.accumulo.config.file", "accumulo-site.xml");
+ if (System.getenv("ACCUMULO_CONF_DIR") != null) {
+ // accumulo conf dir should be set
+ SITE_CONF = System.getenv("ACCUMULO_CONF_DIR") + "/" + configFile;
+ } else if (System.getenv("ACCUMULO_HOME") != null) {
+ // if no accumulo conf dir, try accumulo home default
+ SITE_CONF = System.getenv("ACCUMULO_HOME") + "/conf/" + configFile;
+ } else {
+ SITE_CONF = null;
+ }
+ }
+
+ /**
+ * Parses and XML Document for a property node for a <name> with the value propertyName if it finds one the function return that property's value for its
+ * <value> node. If not found the function will return null
+ *
+ * @param d
+ * XMLDocument to search through
+ * @param propertyName
+ */
+ private static String getAccumuloClassPathStrings(Document d, String propertyName) {
+ NodeList pnodes = d.getElementsByTagName("property");
+ for (int i = pnodes.getLength() - 1; i >= 0; i--) {
+ Element current_property = (Element) pnodes.item(i);
+ Node cname = current_property.getElementsByTagName("name").item(0);
+ if (cname != null && cname.getTextContent().compareTo(propertyName) == 0) {
+ Node cvalue = current_property.getElementsByTagName("value").item(0);
+ if (cvalue != null) {
+ return cvalue.getTextContent();
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Looks for the site configuration file for Accumulo and if it has a property for propertyName return it otherwise returns defaultValue Should throw an
+ * exception if the default configuration can not be read;
+ *
+ * @param propertyName
+ * Name of the property to pull
+ * @param defaultValue
+ * Value to default to if not found.
+ * @return site or default class path String
+ */
+
+ public static String getAccumuloString(String propertyName, String defaultValue) {
+
+ try {
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ String site_classpath_string = null;
+ try {
+ Document site_conf = db.parse(SITE_CONF);
+ site_classpath_string = getAccumuloClassPathStrings(site_conf, propertyName);
+ } catch (Exception e) {
+ /* we don't care because this is optional and we can use defaults */
+ }
+ if (site_classpath_string != null)
+ return site_classpath_string;
+ return defaultValue;
+ } catch (Exception e) {
+ throw new IllegalStateException("ClassPath Strings Lookup failed", e);
+ }
+ }
+
+ /**
+ * Replace environment variables in the classpath string with their actual value
- *
- * @param classpath
- * @param env
+ */
+ public static String replaceEnvVars(String classpath, Map<String,String> env) {
+ Pattern envPat = Pattern.compile("\\$[A-Za-z][a-zA-Z0-9_]*");
+ Matcher envMatcher = envPat.matcher(classpath);
+ while (envMatcher.find(0)) {
+ // name comes after the '$'
+ String varName = envMatcher.group().substring(1);
+ String varValue = env.get(varName);
+ if (varValue == null) {
+ varValue = "";
+ }
+ classpath = (classpath.substring(0, envMatcher.start()) + varValue + classpath.substring(envMatcher.end()));
+ envMatcher.reset(classpath);
+ }
+ return classpath;
+ }
+
+ /**
+ * Populate the list of URLs with the items in the classpath string
+ *
+ * @param classpath
+ * @param urls
+ * @throws MalformedURLException
+ */
+ private static void addUrl(String classpath, ArrayList<URL> urls) throws MalformedURLException {
+ classpath = classpath.trim();
+ if (classpath.length() == 0)
+ return;
+
+ classpath = replaceEnvVars(classpath, System.getenv());
+
+ // Try to make a URI out of the classpath
+ URI uri = null;
+ try {
+ uri = new URI(classpath);
+ } catch (URISyntaxException e) {
+ // Not a valid URI
+ }
+
+ if (null == uri || !uri.isAbsolute() || (null != uri.getScheme() && uri.getScheme().equals("file://"))) {
+ // Then treat this URI as a File.
+ // This checks to see if the url string is a dir if it expand and get all jars in that directory
+ final File extDir = new File(classpath);
+ if (extDir.isDirectory())
+ urls.add(extDir.toURI().toURL());
+ else {
+ if (extDir.getParentFile() != null) {
+ File[] extJars = extDir.getParentFile().listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return name.matches("^" + extDir.getName());
+ }
+ });
+ if (extJars != null && extJars.length > 0) {
+ for (File jar : extJars)
+ urls.add(jar.toURI().toURL());
+ } else {
+ log.debug("ignoring classpath entry " + classpath);
+ }
+ } else {
+ log.debug("ignoring classpath entry " + classpath);
+ }
+ }
+ } else {
+ urls.add(uri.toURL());
+ }
+
+ }
+
+ private static ArrayList<URL> findAccumuloURLs() throws IOException {
+ String cp = getAccumuloString(AccumuloClassLoader.CLASSPATH_PROPERTY_NAME, AccumuloClassLoader.ACCUMULO_CLASSPATH_VALUE);
+ if (cp == null)
+ return new ArrayList<URL>();
+ String[] cps = replaceEnvVars(cp, System.getenv()).split(",");
+ ArrayList<URL> urls = new ArrayList<URL>();
+ for (String classpath : cps) {
+ if (!classpath.startsWith("#")) {
+ addUrl(classpath, urls);
+ }
+ }
+ return urls;
+ }
+
+ public static synchronized ClassLoader getClassLoader() throws IOException {
+ if (classloader == null) {
+ ArrayList<URL> urls = findAccumuloURLs();
+
+ ClassLoader parentClassLoader = AccumuloClassLoader.class.getClassLoader();
+
+ log.debug("Create 2nd tier ClassLoader using URLs: " + urls.toString());
+ URLClassLoader aClassLoader = new URLClassLoader(urls.toArray(new URL[urls.size()]), parentClassLoader) {
+ @Override
+ protected synchronized Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
+
+ if (name.startsWith("org.apache.accumulo.start.classloader.vfs")) {
+ Class<?> c = findLoadedClass(name);
+ if (c == null) {
+ try {
+ // try finding this class here instead of parent
+ findClass(name);
+ } catch (ClassNotFoundException e) {}
+ }
+ }
+ return super.loadClass(name, resolve);
+ }
+ };
+ classloader = aClassLoader;
+ }
+
+ return classloader;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
----------------------------------------------------------------------
diff --cc start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
index 7742cbe,0000000..e1ff55e
mode 100644,000000..100644
--- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
+++ b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
@@@ -1,207 -1,0 +1,205 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.start.classloader.vfs;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileSystemManager;
+
+public class ContextManager {
+
+ // there is a lock per context so that one context can initialize w/o blocking another context
+ private class Context {
+ AccumuloReloadingVFSClassLoader loader;
+ ContextConfig cconfig;
+ boolean closed = false;
+
+ Context(ContextConfig cconfig) {
+ this.cconfig = cconfig;
+ }
+
+ synchronized ClassLoader getClassLoader() throws FileSystemException {
+ if (closed)
+ return null;
+
+ if (loader == null) {
+ loader = new AccumuloReloadingVFSClassLoader(cconfig.uris, vfs, parent, cconfig.preDelegation);
+ }
+
+ return loader.getClassLoader();
+ }
+
+ synchronized void close() {
+ closed = true;
+ loader.close();
+ loader = null;
+ }
+ }
+
+ private Map<String,Context> contexts = new HashMap<String,Context>();
+
+ private volatile ContextsConfig config;
+ private FileSystemManager vfs;
+ private ReloadingClassLoader parent;
+
+ ContextManager(FileSystemManager vfs, ReloadingClassLoader parent) {
+ this.vfs = vfs;
+ this.parent = parent;
+ }
+
+ public static class ContextConfig {
+ String uris;
+ boolean preDelegation;
+
+ public ContextConfig(String uris, boolean preDelegation) {
+ this.uris = uris;
+ this.preDelegation = preDelegation;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof ContextConfig) {
+ ContextConfig oc = (ContextConfig) o;
+
+ return uris.equals(oc.uris) && preDelegation == oc.preDelegation;
+ }
+
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return uris.hashCode() + (preDelegation ? Boolean.TRUE : Boolean.FALSE).hashCode();
+ }
+ }
+
+ public interface ContextsConfig {
+ ContextConfig getContextConfig(String context);
+ }
+
+ public static class DefaultContextsConfig implements ContextsConfig {
+
+ private Iterable<Entry<String,String>> config;
+
+ public DefaultContextsConfig(Iterable<Entry<String,String>> config) {
+ this.config = config;
+ }
+
+ @Override
+ public ContextConfig getContextConfig(String context) {
+
+ String key = AccumuloVFSClassLoader.VFS_CONTEXT_CLASSPATH_PROPERTY + context;
+
+ String uris = null;
+ boolean preDelegate = true;
+
+ Iterator<Entry<String,String>> iter = config.iterator();
+ while (iter.hasNext()) {
+ Entry<String,String> entry = iter.next();
+ if (entry.getKey().equals(key)) {
+ uris = entry.getValue();
+ }
+
+ if (entry.getKey().equals(key + ".delegation") && entry.getValue().trim().equalsIgnoreCase("post")) {
+ preDelegate = false;
+ }
+ }
+
+ if (uris != null)
+ return new ContextConfig(uris, preDelegate);
+
+ return null;
+ }
+ }
+
+ /**
+ * configuration must be injected for ContextManager to work
- *
- * @param config
+ */
+ public synchronized void setContextConfig(ContextsConfig config) {
+ if (this.config != null)
+ throw new IllegalStateException("Context manager config already set");
+ this.config = config;
+ }
+
+ public ClassLoader getClassLoader(String contextName) throws FileSystemException {
+
+ ContextConfig cconfig = config.getContextConfig(contextName);
+
+ if (cconfig == null)
+ throw new IllegalArgumentException("Unknown context " + contextName);
+
+ Context context = null;
+ Context contextToClose = null;
+
+ synchronized (this) {
+ // only manipulate internal data structs in this sync block... avoid creating or closing classloader, reading config, etc... basically avoid operations
+ // that may block
+ context = contexts.get(contextName);
+
+ if (context == null) {
+ context = new Context(cconfig);
+ contexts.put(contextName, context);
+ } else if (!context.cconfig.equals(cconfig)) {
+ contextToClose = context;
+ context = new Context(cconfig);
+ contexts.put(contextName, context);
+ }
+ }
+
+ if (contextToClose != null)
+ contextToClose.close();
+
+ ClassLoader loader = context.getClassLoader();
+ if (loader == null) {
+ // ooppss, context was closed by another thread, try again
+ return getClassLoader(contextName);
+ }
+
+ return loader;
+
+ }
+
+ public <U> Class<? extends U> loadClass(String context, String classname, Class<U> extension) throws ClassNotFoundException {
+ try {
+ return getClassLoader(context).loadClass(classname).asSubclass(extension);
+ } catch (IOException e) {
+ throw new ClassNotFoundException("IO Error loading class " + classname, e);
+ }
+ }
+
+ public void removeUnusedContexts(Set<String> inUse) {
+
+ Map<String,Context> unused;
+
+ synchronized (this) {
+ unused = new HashMap<String,Context>(contexts);
+ unused.keySet().removeAll(inUse);
+ contexts.keySet().removeAll(unused.keySet());
+ }
+
+ for (Context context : unused.values()) {
+ // close outside of lock
+ context.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
----------------------------------------------------------------------
diff --cc start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
index 0a6931f,0000000..277c741
mode 100644,000000..100644
--- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
+++ b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
@@@ -1,52 -1,0 +1,47 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.start.classloader.vfs;
+
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileSystemManager;
+import org.apache.commons.vfs2.impl.VFSClassLoader;
+
+/**
+ *
+ */
+public class PostDelegatingVFSClassLoader extends VFSClassLoader {
+
- /**
- * @param files
- * @param manager
- * @param parent
- * @throws FileSystemException
- */
+ public PostDelegatingVFSClassLoader(FileObject[] files, FileSystemManager manager, ClassLoader parent) throws FileSystemException {
+ super(files, manager, parent);
+ }
+
++ @Override
+ protected synchronized Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
+ Class<?> c = findLoadedClass(name);
+ if (c == null) {
+ try {
+ // try finding this class here instead of parent
+ findClass(name);
+ } catch (ClassNotFoundException e) {
+
+ }
+ }
+ return super.loadClass(name, resolve);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java
----------------------------------------------------------------------
diff --cc start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java
index 92b2720,0000000..104ea09
mode 100644,000000..100644
--- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java
+++ b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java
@@@ -1,164 -1,0 +1,159 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.start.classloader.vfs.providers;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.vfs2.CacheStrategy;
+import org.apache.commons.vfs2.Capability;
+import org.apache.commons.vfs2.FileName;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileSystemOptions;
+import org.apache.commons.vfs2.provider.AbstractFileName;
+import org.apache.commons.vfs2.provider.AbstractFileSystem;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * A VFS FileSystem that interacts with HDFS.
+ *
+ * @since 2.1
+ */
+public class HdfsFileSystem extends AbstractFileSystem
+{
+ private static final Log log = LogFactory.getLog(HdfsFileSystem.class);
+
+ private FileSystem fs;
+
- /**
- *
- * @param rootName
- * @param fileSystemOptions
- */
+ protected HdfsFileSystem(final FileName rootName, final FileSystemOptions fileSystemOptions)
+ {
+ super(rootName, null, fileSystemOptions);
+ }
+
+ /**
+ * @see org.apache.commons.vfs2.provider.AbstractFileSystem#addCapabilities(java.util.Collection)
+ */
+ @Override
+ protected void addCapabilities(final Collection<Capability> capabilities)
+ {
+ capabilities.addAll(HdfsFileProvider.CAPABILITIES);
+ }
+
+ /**
+ * @see org.apache.commons.vfs2.provider.AbstractFileSystem#close()
+ */
+ @Override
+ public void close()
+ {
+ try
+ {
+ if (null != fs)
+ {
+ fs.close();
+ }
+ }
+ catch (final IOException e)
+ {
+ throw new RuntimeException("Error closing HDFS client", e);
+ }
+ super.close();
+ }
+
+ /**
+ * @see org.apache.commons.vfs2.provider.AbstractFileSystem#createFile(org.apache.commons.vfs2.provider.AbstractFileName)
+ */
+ @Override
+ protected FileObject createFile(final AbstractFileName name) throws Exception
+ {
+ throw new FileSystemException("Operation not supported");
+ }
+
+ /**
+ * @see org.apache.commons.vfs2.provider.AbstractFileSystem#resolveFile(org.apache.commons.vfs2.FileName)
+ */
+ @Override
+ public FileObject resolveFile(final FileName name) throws FileSystemException
+ {
+
+ synchronized (this)
+ {
+ if (null == this.fs)
+ {
+ final String hdfsUri = name.getRootURI();
+ final Configuration conf = new Configuration(true);
+ conf.set(org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY, hdfsUri);
+ this.fs = null;
+ try
+ {
+ fs = org.apache.hadoop.fs.FileSystem.get(conf);
+ }
+ catch (final IOException e)
+ {
+ log.error("Error connecting to filesystem " + hdfsUri, e);
+ throw new FileSystemException("Error connecting to filesystem " + hdfsUri, e);
+ }
+ }
+ }
+
+ boolean useCache = (null != getContext().getFileSystemManager().getFilesCache());
+ FileObject file;
+ if (useCache)
+ {
+ file = this.getFileFromCache(name);
+ }
+ else
+ {
+ file = null;
+ }
+ if (null == file)
+ {
+ String path = null;
+ try
+ {
+ path = URLDecoder.decode(name.getPath(), "UTF-8");
+ }
+ catch (final UnsupportedEncodingException e)
+ {
+ path = name.getPath();
+ }
+ final Path filePath = new Path(path);
+ file = new HdfsFileObject((AbstractFileName) name, this, fs, filePath);
+ if (useCache)
+ {
+ this.putFileToCache(file);
+ }
+
+ }
+
+ /**
+ * resync the file information if requested
+ */
+ if (getFileSystemManager().getCacheStrategy().equals(CacheStrategy.ON_RESOLVE)) {
+ file.refresh();
+ }
+
+ return file;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/GetMasterStats.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/GetMasterStats.java
index 65cf80c,0000000..7b1313a
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/GetMasterStats.java
+++ b/test/src/main/java/org/apache/accumulo/test/GetMasterStats.java
@@@ -1,129 -1,0 +1,120 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
- import java.io.IOException;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.accumulo.core.client.impl.MasterClient;
- import org.apache.accumulo.core.master.MasterNotRunningException;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
+import org.apache.accumulo.core.master.thrift.RecoveryStatus;
+import org.apache.accumulo.core.master.thrift.TableInfo;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.monitor.Monitor;
+import org.apache.accumulo.server.security.SecurityConstants;
- import org.apache.thrift.transport.TTransportException;
+
+public class GetMasterStats {
- /**
- * @param args
- * @throws MasterNotRunningException
- * @throws IOException
- * @throws TTransportException
- */
+ public static void main(String[] args) throws Exception {
+ MasterClientService.Iface client = null;
+ MasterMonitorInfo stats = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(HdfsZooInstance.getInstance());
+ stats = client.getMasterStats(Tracer.traceInfo(), SecurityConstants.getSystemCredentials());
+ } finally {
+ if (client != null)
+ MasterClient.close(client);
+ }
+ out(0, "State: " + stats.state.name());
+ out(0, "Goal State: " + stats.goalState.name());
+ if (stats.serversShuttingDown != null && stats.serversShuttingDown.size() > 0) {
+ out(0, "Servers to shutdown");
+ for (String server : stats.serversShuttingDown) {
+ out(1, "%s", server);
+ }
+ }
+ out(0, "Unassigned tablets: %d", stats.unassignedTablets);
+ if (stats.badTServers != null && stats.badTServers.size() > 0) {
+ out(0, "Bad servers");
+
+ for (Entry<String,Byte> entry : stats.badTServers.entrySet()) {
+ out(1, "%s: %d", entry.getKey(), (int) entry.getValue());
+ }
+ }
+ if (stats.tableMap != null && stats.tableMap.size() > 0) {
+ out(0, "Tables");
+ for (Entry<String,TableInfo> entry : stats.tableMap.entrySet()) {
+ TableInfo v = entry.getValue();
+ out(1, "%s", entry.getKey());
+ out(2, "Records: %d", v.recs);
+ out(2, "Records in Memory: %d", v.recsInMemory);
+ out(2, "Tablets: %d", v.tablets);
+ out(2, "Online Tablets: %d", v.onlineTablets);
+ out(2, "Ingest Rate: %.2f", v.ingestRate);
+ out(2, "Query Rate: %.2f", v.queryRate);
+ }
+ }
+ if (stats.tServerInfo != null && stats.tServerInfo.size() > 0) {
+ out(0, "Tablet Servers");
+ long now = System.currentTimeMillis();
+ for (TabletServerStatus server : stats.tServerInfo) {
+ TableInfo summary = Monitor.summarizeTableStats(server);
+ out(1, "Name: %s", server.name);
+ out(2, "Ingest: %.2f", summary.ingestRate);
+ out(2, "Last Contact: %s", server.lastContact);
+ out(2, "OS Load Average: %.2f", server.osLoad);
+ out(2, "Queries: %.2f", summary.queryRate);
+ out(2, "Time Difference: %.1f", ((now - server.lastContact) / 1000.));
+ out(2, "Total Records: %d", summary.recs);
+ out(2, "Lookups: %d", server.lookups);
+ if (server.holdTime > 0)
+ out(2, "Hold Time: %d", server.holdTime);
+ if (server.tableMap != null && server.tableMap.size() > 0) {
+ out(2, "Tables");
+ for (Entry<String,TableInfo> status : server.tableMap.entrySet()) {
+ TableInfo info = status.getValue();
+ out(3, "Table: %s", status.getKey());
+ out(4, "Tablets: %d", info.onlineTablets);
+ out(4, "Records: %d", info.recs);
+ out(4, "Records in Memory: %d", info.recsInMemory);
+ out(4, "Ingest: %.2f", info.ingestRate);
+ out(4, "Queries: %.2f", info.queryRate);
+ out(4, "Major Compacting: %d", info.majors == null ? 0 : info.majors.running);
+ out(4, "Queued for Major Compaction: %d", info.majors == null ? 0 : info.majors.queued);
+ out(4, "Minor Compacting: %d", info.minors == null ? 0 : info.minors.running);
+ out(4, "Queued for Minor Compaction: %d", info.minors == null ? 0 : info.minors.queued);
+ }
+ }
+ out(2, "Recoveries: %d", server.logSorts.size());
+ for (RecoveryStatus sort : server.logSorts) {
+ out(3, "File: %s", sort.name);
+ out(3, "Progress: %.2f%%", sort.progress * 100);
+ out(3, "Time running: %s", sort.runtime / 1000.);
+ }
+ }
+ }
+ }
+
+ private static void out(int indent, String string, Object... args) {
+ for (int i = 0; i < indent; i++) {
+ System.out.print(" ");
+ }
+ System.out.println(String.format(string, args));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
index 73b73f4,0000000..16e7a98
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
+++ b/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
@@@ -1,198 -1,0 +1,195 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.FastFormat;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.tabletserver.NativeMap;
+import org.apache.hadoop.io.Text;
+
+public class NativeMapPerformanceTest {
+
+ private static final byte ROW_PREFIX[] = new byte[] {'r'};
+ private static final byte COL_PREFIX[] = new byte[] {'c'};
+
+ static Key nk(int r, int c) {
+ return new Key(new Text(FastFormat.toZeroPaddedString(r, 9, 10, ROW_PREFIX)), new Text(FastFormat.toZeroPaddedString(c, 6, 10, COL_PREFIX)));
+ }
+
+ static Mutation nm(int r) {
+ return new Mutation(new Text(FastFormat.toZeroPaddedString(r, 9, 10, ROW_PREFIX)));
+ }
+
+ static Text ET = new Text();
+
+ private static void pc(Mutation m, int c, Value v) {
+ m.put(new Text(FastFormat.toZeroPaddedString(c, 6, 10, COL_PREFIX)), ET, Long.MAX_VALUE, v);
+ }
+
+ static void runPerformanceTest(int numRows, int numCols, int numLookups, String mapType) {
+
+ SortedMap<Key,Value> tm = null;
+ NativeMap nm = null;
+
+ if (mapType.equals("SKIP_LIST"))
+ tm = new ConcurrentSkipListMap<Key,Value>();
+ else if (mapType.equals("TREE_MAP"))
+ tm = Collections.synchronizedSortedMap(new TreeMap<Key,Value>());
+ else if (mapType.equals("NATIVE_MAP"))
+ nm = new NativeMap();
+ else
+ throw new IllegalArgumentException(" map type must be SKIP_LIST, TREE_MAP, or NATIVE_MAP");
+
+ Random rand = new Random(19);
+
+ // puts
+ long tps = System.currentTimeMillis();
+
+ if (nm != null) {
+ for (int i = 0; i < numRows; i++) {
+ int row = rand.nextInt(1000000000);
+ Mutation m = nm(row);
+ for (int j = 0; j < numCols; j++) {
+ int col = rand.nextInt(1000000);
+ Value val = new Value("test".getBytes(Constants.UTF8));
+ pc(m, col, val);
+ }
+ nm.mutate(m, i);
+ }
+ } else {
+ for (int i = 0; i < numRows; i++) {
+ int row = rand.nextInt(1000000000);
+ for (int j = 0; j < numCols; j++) {
+ int col = rand.nextInt(1000000);
+ Key key = nk(row, col);
+ Value val = new Value("test".getBytes(Constants.UTF8));
+ tm.put(key, val);
+ }
+ }
+ }
+
+ long tpe = System.currentTimeMillis();
+
+ // Iteration
+ Iterator<Entry<Key,Value>> iter;
+ if (nm != null) {
+ iter = nm.iterator();
+ } else {
+ iter = tm.entrySet().iterator();
+ }
+
+ long tis = System.currentTimeMillis();
+
+ while (iter.hasNext()) {
+ iter.next();
+ }
+
+ long tie = System.currentTimeMillis();
+
+ rand = new Random(19);
+ int rowsToLookup[] = new int[numLookups];
+ int colsToLookup[] = new int[numLookups];
+ for (int i = 0; i < Math.min(numLookups, numRows); i++) {
+ int row = rand.nextInt(1000000000);
+ int col = -1;
+ for (int j = 0; j < numCols; j++) {
+ col = rand.nextInt(1000000);
+ }
+
+ rowsToLookup[i] = row;
+ colsToLookup[i] = col;
+ }
+
+ // get
+
+ long tgs = System.currentTimeMillis();
+ if (nm != null) {
+ for (int i = 0; i < numLookups; i++) {
+ Key key = nk(rowsToLookup[i], colsToLookup[i]);
+ if (nm.get(key) == null) {
+ throw new RuntimeException("Did not find " + rowsToLookup[i] + " " + colsToLookup[i] + " " + i);
+ }
+ }
+ } else {
+ for (int i = 0; i < numLookups; i++) {
+ Key key = nk(rowsToLookup[i], colsToLookup[i]);
+ if (tm.get(key) == null) {
+ throw new RuntimeException("Did not find " + rowsToLookup[i] + " " + colsToLookup[i] + " " + i);
+ }
+ }
+ }
+ long tge = System.currentTimeMillis();
+
+ long memUsed = 0;
+ if (nm != null) {
+ memUsed = nm.getMemoryUsed();
+ }
+
+ int size = (nm == null ? tm.size() : nm.size());
+
+ // delete
+ long tds = System.currentTimeMillis();
+
+ if (nm != null)
+ nm.delete();
+
+ long tde = System.currentTimeMillis();
+
+ if (tm != null)
+ tm.clear();
+
+ System.gc();
+ System.gc();
+ System.gc();
+ System.gc();
+
+ UtilWaitThread.sleep(3000);
+
+ System.out.printf("mapType:%10s put rate:%,6.2f scan rate:%,6.2f get rate:%,6.2f delete time : %6.2f mem : %,d%n", "" + mapType, (numRows * numCols)
+ / ((tpe - tps) / 1000.0), (size) / ((tie - tis) / 1000.0), numLookups / ((tge - tgs) / 1000.0), (tde - tds) / 1000.0, memUsed);
+
+ }
+
- /**
- * @param args
- */
+ public static void main(String[] args) {
+
+ if (args.length != 3) {
+ throw new IllegalArgumentException("Usage : " + NativeMapPerformanceTest.class.getName() + " <map type> <rows> <columns>");
+ }
+
+ String mapType = args[0];
+ int rows = Integer.parseInt(args[1]);
+ int cols = Integer.parseInt(args[2]);
+
+ runPerformanceTest(rows, cols, 10000, mapType);
+ runPerformanceTest(rows, cols, 10000, mapType);
+ runPerformanceTest(rows, cols, 10000, mapType);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
index 8411c86,0000000..5115541
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
+++ b/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
@@@ -1,277 -1,0 +1,274 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.OpTimer;
+import org.apache.accumulo.server.tabletserver.NativeMap;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+public class NativeMapStressTest {
+
+ private static final Logger log = Logger.getLogger(NativeMapStressTest.class);
+
- /**
- * @param args
- */
+ public static void main(String[] args) {
+ testLotsOfMapDeletes(true);
+ testLotsOfMapDeletes(false);
+ testLotsOfOverwrites();
+ testLotsOfGetsAndScans();
+ }
+
+ private static void put(NativeMap nm, String row, String val, int mc) {
+ Mutation m = new Mutation(new Text(row));
+ m.put(new Text(), new Text(), Long.MAX_VALUE, new Value(val.getBytes(Constants.UTF8)));
+ nm.mutate(m, mc);
+ }
+
+ private static void testLotsOfGetsAndScans() {
+
+ ArrayList<Thread> threads = new ArrayList<Thread>();
+
+ final int numThreads = 8;
+ final int totalGets = 100000000;
+ final int mapSizePerThread = (int) (4000000 / (double) numThreads);
+ final int getsPerThread = (int) (totalGets / (double) numThreads);
+
+ for (int tCount = 0; tCount < numThreads; tCount++) {
+ Runnable r = new Runnable() {
+ @Override
+ public void run() {
+ NativeMap nm = new NativeMap();
+
+ Random r = new Random();
+
+ OpTimer opTimer = new OpTimer(log, Level.INFO);
+
+ opTimer.start("Creating map of size " + mapSizePerThread);
+
+ for (int i = 0; i < mapSizePerThread; i++) {
+ String row = String.format("r%08d", i);
+ String val = row + "v";
+ put(nm, row, val, i);
+ }
+
+ opTimer.stop("Created map of size " + nm.size() + " in %DURATION%");
+
+ opTimer.start("Doing " + getsPerThread + " gets()");
+
+ for (int i = 0; i < getsPerThread; i++) {
+ String row = String.format("r%08d", r.nextInt(mapSizePerThread));
+ String val = row + "v";
+
+ Value value = nm.get(new Key(new Text(row)));
+ if (value == null || !value.toString().equals(val)) {
+ log.error("nm.get(" + row + ") failed");
+ }
+ }
+
+ opTimer.stop("Finished " + getsPerThread + " gets in %DURATION%");
+
+ int scanned = 0;
+
+ opTimer.start("Doing " + getsPerThread + " random iterations");
+
+ for (int i = 0; i < getsPerThread; i++) {
+ int startRow = r.nextInt(mapSizePerThread);
+ String row = String.format("r%08d", startRow);
+
+ Iterator<Entry<Key,Value>> iter = nm.iterator(new Key(new Text(row)));
+
+ int count = 0;
+
+ while (iter.hasNext() && count < 10) {
+ String row2 = String.format("r%08d", startRow + count);
+ String val2 = row2 + "v";
+
+ Entry<Key,Value> entry = iter.next();
+ if (!entry.getValue().toString().equals(val2) || !entry.getKey().equals(new Key(new Text(row2)))) {
+ log.error("nm.iter(" + row2 + ") failed row = " + row + " count = " + count + " row2 = " + row + " val2 = " + val2);
+ }
+
+ count++;
+ }
+
+ scanned += count;
+ }
+
+ opTimer.stop("Finished " + getsPerThread + " random iterations (scanned = " + scanned + ") in %DURATION%");
+
+ nm.delete();
+ }
+ };
+
+ Thread t = new Thread(r);
+ t.start();
+
+ threads.add(t);
+ }
+
+ for (Thread thread : threads) {
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private static void testLotsOfMapDeletes(final boolean doRemoves) {
+ final int numThreads = 8;
+ final int rowRange = 10000;
+ final int mapsPerThread = 50;
+ final int totalInserts = 100000000;
+ final int insertsPerMapPerThread = (int) (totalInserts / (double) numThreads / mapsPerThread);
+
+ System.out.println("insertsPerMapPerThread " + insertsPerMapPerThread);
+
+ ArrayList<Thread> threads = new ArrayList<Thread>();
+
+ for (int i = 0; i < numThreads; i++) {
+ Runnable r = new Runnable() {
+ @Override
+ public void run() {
+
+ int inserts = 0;
+ int removes = 0;
+
+ for (int i = 0; i < mapsPerThread; i++) {
+
+ NativeMap nm = new NativeMap();
+
+ for (int j = 0; j < insertsPerMapPerThread; j++) {
+ String row = String.format("r%08d", j % rowRange);
+ String val = row + "v";
+ put(nm, row, val, j);
+ inserts++;
+ }
+
+ if (doRemoves) {
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+ while (iter.hasNext()) {
+ iter.next();
+ iter.remove();
+ removes++;
+ }
+ }
+
+ nm.delete();
+ }
+
+ System.out.println("inserts " + inserts + " removes " + removes + " " + Thread.currentThread().getName());
+ }
+ };
+
+ Thread t = new Thread(r);
+ t.start();
+
+ threads.add(t);
+ }
+
+ for (Thread thread : threads) {
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private static void testLotsOfOverwrites() {
+ final Map<Integer,NativeMap> nativeMaps = new HashMap<Integer,NativeMap>();
+
+ int numThreads = 8;
+ final int insertsPerThread = (int) (100000000 / (double) numThreads);
+ final int rowRange = 10000;
+ final int numMaps = 50;
+
+ ArrayList<Thread> threads = new ArrayList<Thread>();
+
+ for (int i = 0; i < numThreads; i++) {
+ Runnable r = new Runnable() {
+ @Override
+ public void run() {
+ Random r = new Random();
+ int inserts = 0;
+
+ for (int i = 0; i < insertsPerThread / 100.0; i++) {
+ int map = r.nextInt(numMaps);
+
+ NativeMap nm;
+
+ synchronized (nativeMaps) {
+ nm = nativeMaps.get(map);
+ if (nm == null) {
+ nm = new NativeMap();
+ nativeMaps.put(map, nm);
+
+ }
+ }
+
+ synchronized (nm) {
+ for (int j = 0; j < 100; j++) {
+ String row = String.format("r%08d", r.nextInt(rowRange));
+ String val = row + "v";
+ put(nm, row, val, j);
+ inserts++;
+ }
+ }
+ }
+
+ System.out.println("inserts " + inserts + " " + Thread.currentThread().getName());
+ }
+ };
+
+ Thread t = new Thread(r);
+ t.start();
+
+ threads.add(t);
+ }
+
+ for (Thread thread : threads) {
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ }
+ }
+
+ Set<Entry<Integer,NativeMap>> es = nativeMaps.entrySet();
+ for (Entry<Integer,NativeMap> entry : es) {
+ entry.getValue().delete();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
index a35ca66,0000000..0d52f12
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
@@@ -1,181 -1,0 +1,180 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.continuous;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
+import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.server.util.reflection.CounterUtils;
+import org.apache.accumulo.test.continuous.ContinuousIngest.BaseOpts;
+import org.apache.accumulo.test.continuous.ContinuousIngest.ShortConverter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.validators.PositiveInteger;
+
+/**
+ * A map only job that reads a table created by continuous ingest and creates doubly linked list. This map reduce job tests the ability of a map only job to
+ * read and write to accumulo at the same time. This map reduce job mutates the table in such a way that it should not create any undefined nodes.
+ *
+ */
+public class ContinuousMoru extends Configured implements Tool {
+ private static final String PREFIX = ContinuousMoru.class.getSimpleName() + ".";
+ private static final String MAX_CQ = PREFIX + "MAX_CQ";
+ private static final String MAX_CF = PREFIX + "MAX_CF";
+ private static final String MAX = PREFIX + "MAX";
+ private static final String MIN = PREFIX + "MIN";
+ private static final String CI_ID = PREFIX + "CI_ID";
+
+ static enum Counts {
+ SELF_READ;
+ }
+
+ public static class CMapper extends Mapper<Key,Value,Text,Mutation> {
+
+ private short max_cf;
+ private short max_cq;
+ private Random random;
+ private String ingestInstanceId;
+ private byte[] iiId;
+ private long count;
+
+ private static final ColumnVisibility EMPTY_VIS = new ColumnVisibility();
+
+ @Override
+ public void setup(Context context) throws IOException, InterruptedException {
+ int max_cf = context.getConfiguration().getInt(MAX_CF, -1);
+ int max_cq = context.getConfiguration().getInt(MAX_CQ, -1);
+
+ if (max_cf > Short.MAX_VALUE || max_cq > Short.MAX_VALUE)
+ throw new IllegalArgumentException();
+
+ this.max_cf = (short) max_cf;
+ this.max_cq = (short) max_cq;
+
+ random = new Random();
+ ingestInstanceId = context.getConfiguration().get(CI_ID);
+ iiId = ingestInstanceId.getBytes(Constants.UTF8);
+
+ count = 0;
+ }
+
+ @Override
+ public void map(Key key, Value data, Context context) throws IOException, InterruptedException {
+
+ ContinuousWalk.validate(key, data);
+
+ if (WritableComparator.compareBytes(iiId, 0, iiId.length, data.get(), 0, iiId.length) != 0) {
+ // only rewrite data not written by this M/R job
+ byte[] val = data.get();
+
+ int offset = ContinuousWalk.getPrevRowOffset(val);
+ if (offset > 0) {
+ long rowLong = Long.parseLong(new String(val, offset, 16, Constants.UTF8), 16);
+ Mutation m = ContinuousIngest.genMutation(rowLong, random.nextInt(max_cf), random.nextInt(max_cq), EMPTY_VIS, iiId, count++, key.getRowData()
+ .toArray(), random, true);
+ context.write(null, m);
+ }
+
+ } else {
+ CounterUtils.increment(context.getCounter(Counts.SELF_READ));
+ }
+ }
+ }
+
+ static class Opts extends BaseOpts {
+ @Parameter(names = "--maxColF", description = "maximum column family value to use", converter=ShortConverter.class)
+ short maxColF = Short.MAX_VALUE;
+
+ @Parameter(names = "--maxColQ", description = "maximum column qualifier value to use", converter=ShortConverter.class)
+ short maxColQ = Short.MAX_VALUE;
+
+ @Parameter(names = "--maxMappers", description = "the maximum number of mappers to use", required = true, validateWith = PositiveInteger.class)
+ int maxMaps = 0;
+ }
+
+ @Override
+ public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException, AccumuloSecurityException {
+ Opts opts = new Opts();
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(ContinuousMoru.class.getName(), args, bwOpts);
+
+ Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+ job.setJarByClass(this.getClass());
+
+ job.setInputFormatClass(AccumuloInputFormat.class);
+ opts.setAccumuloConfigs(job);
+
+ // set up ranges
+ try {
+ Set<Range> ranges = opts.getConnector().tableOperations().splitRangeByTablets(opts.getTableName(), new Range(), opts.maxMaps);
+ AccumuloInputFormat.setRanges(job, ranges);
+ AccumuloInputFormat.setAutoAdjustRanges(job, false);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ job.setMapperClass(CMapper.class);
+
+ job.setNumReduceTasks(0);
+
+ job.setOutputFormatClass(AccumuloOutputFormat.class);
+ AccumuloOutputFormat.setBatchWriterOptions(job, bwOpts.getBatchWriterConfig());
+
+ Configuration conf = job.getConfiguration();
+ conf.setLong(MIN, opts.min);
+ conf.setLong(MAX, opts.max);
+ conf.setInt(MAX_CF, opts.maxColF);
+ conf.setInt(MAX_CQ, opts.maxColQ);
+ conf.set(CI_ID, UUID.randomUUID().toString());
+
+ job.waitForCompletion(true);
+ opts.stopTracing();
+ return job.isSuccessful() ? 0 : 1;
+ }
+
+ /**
+ *
+ * @param args
+ * instanceName zookeepers username password table columns outputpath
- * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(CachedConfiguration.getInstance(), new ContinuousMoru(), args);
+ if (res != 0)
+ System.exit(res);
+ }
+}
[49/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
index 448981b,0000000..442f1be
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
@@@ -1,1322 -1,0 +1,1318 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.admin;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableDeletedException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.client.impl.AccumuloServerException;
+import org.apache.accumulo.core.client.impl.ClientExec;
+import org.apache.accumulo.core.client.impl.ClientExecReturn;
+import org.apache.accumulo.core.client.impl.MasterClient;
+import org.apache.accumulo.core.client.impl.ServerClient;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.TabletLocator;
+import org.apache.accumulo.core.client.impl.TabletLocator.TabletLocation;
+import org.apache.accumulo.core.client.impl.thrift.ClientService;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.constraints.Constraint;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.master.thrift.TableOperation;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.LocalityGroupUtil;
+import org.apache.accumulo.core.util.MetadataTable;
+import org.apache.accumulo.core.util.NamingThreadFactory;
+import org.apache.accumulo.core.util.OpTimer;
+import org.apache.accumulo.core.util.StringUtil;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TApplicationException;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransportException;
+
+/**
+ * Provides a class for administering tables
+ *
+ */
+public class TableOperationsImpl extends TableOperationsHelper {
+ private Instance instance;
+ private TCredentials credentials;
+
+ public static final String CLONE_EXCLUDE_PREFIX = "!";
+
+ private static final Logger log = Logger.getLogger(TableOperations.class);
+
+ /**
+ * @param instance
+ * the connection information for this instance
+ * @param credentials
+ * the username/password for this connection
+ */
+ public TableOperationsImpl(Instance instance, TCredentials credentials) {
+ ArgumentChecker.notNull(instance, credentials);
+ this.instance = instance;
+ this.credentials = credentials;
+ }
+
+ /**
+ * Retrieve a list of tables in Accumulo.
+ *
+ * @return List of tables in accumulo
+ */
+ @Override
+ public SortedSet<String> list() {
+ OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Fetching list of tables...");
+ TreeSet<String> tableNames = new TreeSet<String>(Tables.getNameToIdMap(instance).keySet());
+ opTimer.stop("Fetched " + tableNames.size() + " table names in %DURATION%");
+ return tableNames;
+ }
+
+ /**
+ * A method to check if a table exists in Accumulo.
+ *
+ * @param tableName
+ * the name of the table
+ * @return true if the table exists
+ */
+ @Override
+ public boolean exists(String tableName) {
+ ArgumentChecker.notNull(tableName);
+ if (tableName.equals(Constants.METADATA_TABLE_NAME))
+ return true;
+
+ OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Checking if table " + tableName + "exists...");
+ boolean exists = Tables.getNameToIdMap(instance).containsKey(tableName);
+ opTimer.stop("Checked existance of " + exists + " in %DURATION%");
+ return exists;
+ }
+
+ /**
+ * Create a table with no special configuration
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableExistsException
+ * if the table already exists
+ */
+ @Override
+ public void create(String tableName) throws AccumuloException, AccumuloSecurityException, TableExistsException {
+ create(tableName, true, TimeType.MILLIS);
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param limitVersion
+ * Enables/disables the versioning iterator, which will limit the number of Key versions kept.
+ */
+ @Override
+ public void create(String tableName, boolean limitVersion) throws AccumuloException, AccumuloSecurityException, TableExistsException {
+ create(tableName, limitVersion, TimeType.MILLIS);
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param timeType
+ * specifies logical or real-time based time recording for entries in the table
+ * @param limitVersion
+ * Enables/disables the versioning iterator, which will limit the number of Key versions kept.
+ */
+ @Override
+ public void create(String tableName, boolean limitVersion, TimeType timeType) throws AccumuloException, AccumuloSecurityException, TableExistsException {
+ ArgumentChecker.notNull(tableName, timeType);
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(timeType.name().getBytes(Constants.UTF8)));
+
+ Map<String,String> opts = IteratorUtil.generateInitialTableProperties(limitVersion);
+
+ try {
+ doTableOperation(TableOperation.CREATE, args, opts);
+ } catch (TableNotFoundException e1) {
+ // should not happen
+ throw new RuntimeException(e1);
+ }
+ }
+
+ private long beginTableOperation() throws ThriftSecurityException, TException {
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ return client.beginTableOperation(Tracer.traceInfo(), credentials);
+ } catch (TTransportException tte) {
+ log.debug("Failed to call beginTableOperation(), retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ }
+
+ private void executeTableOperation(long opid, TableOperation op, List<ByteBuffer> args, Map<String,String> opts, boolean autoCleanUp)
+ throws ThriftSecurityException, TException, ThriftTableOperationException {
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ client.executeTableOperation(Tracer.traceInfo(), credentials, opid, op, args, opts, autoCleanUp);
+ break;
+ } catch (TTransportException tte) {
+ log.debug("Failed to call executeTableOperation(), retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ }
+
+ private String waitForTableOperation(long opid) throws ThriftSecurityException, TException, ThriftTableOperationException {
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ return client.waitForTableOperation(Tracer.traceInfo(), credentials, opid);
+ } catch (TTransportException tte) {
+ log.debug("Failed to call waitForTableOperation(), retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ }
+
+ private void finishTableOperation(long opid) throws ThriftSecurityException, TException {
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ client.finishTableOperation(Tracer.traceInfo(), credentials, opid);
+ break;
+ } catch (TTransportException tte) {
+ log.debug("Failed to call finishTableOperation(), retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ }
+
+ private String doTableOperation(TableOperation op, List<ByteBuffer> args, Map<String,String> opts) throws AccumuloSecurityException, TableExistsException,
+ TableNotFoundException, AccumuloException {
+ return doTableOperation(op, args, opts, true);
+ }
+
+ private String doTableOperation(TableOperation op, List<ByteBuffer> args, Map<String,String> opts, boolean wait) throws AccumuloSecurityException,
+ TableExistsException, TableNotFoundException, AccumuloException {
+ Long opid = null;
+
+ try {
+ opid = beginTableOperation();
+ executeTableOperation(opid, op, args, opts, !wait);
+ if (!wait) {
+ opid = null;
+ return null;
+ }
+ String ret = waitForTableOperation(opid);
+ Tables.clearCache(instance);
+ return ret;
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (ThriftTableOperationException e) {
+ switch (e.getType()) {
+ case EXISTS:
+ throw new TableExistsException(e);
+ case NOTFOUND:
+ throw new TableNotFoundException(e);
+ case OFFLINE:
+ throw new TableOfflineException(instance, null);
+ case OTHER:
+ default:
+ throw new AccumuloException(e.description, e);
+ }
+ } catch (Exception e) {
+ throw new AccumuloException(e.getMessage(), e);
+ } finally {
+ // always finish table op, even when exception
+ if (opid != null)
+ try {
+ finishTableOperation(opid);
+ } catch (Exception e) {
+ log.warn(e.getMessage(), e);
+ }
+ }
+ }
+
+ private static class SplitEnv {
+ private String tableName;
+ private String tableId;
+ private ExecutorService executor;
+ private CountDownLatch latch;
+ private AtomicReference<Exception> exception;
+
+ SplitEnv(String tableName, String tableId, ExecutorService executor, CountDownLatch latch, AtomicReference<Exception> exception) {
+ this.tableName = tableName;
+ this.tableId = tableId;
+ this.executor = executor;
+ this.latch = latch;
+ this.exception = exception;
+ }
+ }
+
+ private class SplitTask implements Runnable {
+
+ private List<Text> splits;
+ private SplitEnv env;
+
+ SplitTask(SplitEnv env, List<Text> splits) {
+ this.env = env;
+ this.splits = splits;
+ }
+
+ @Override
+ public void run() {
+ try {
+ if (env.exception.get() != null)
+ return;
+
+ if (splits.size() <= 2) {
+ addSplits(env.tableName, new TreeSet<Text>(splits), env.tableId);
+ for (int i = 0; i < splits.size(); i++)
+ env.latch.countDown();
+ return;
+ }
+
+ int mid = splits.size() / 2;
+
+ // split the middle split point to ensure that child task split different tablets and can therefore
+ // run in parallel
+ addSplits(env.tableName, new TreeSet<Text>(splits.subList(mid, mid + 1)), env.tableId);
+ env.latch.countDown();
+
+ env.executor.submit(new SplitTask(env, splits.subList(0, mid)));
+ env.executor.submit(new SplitTask(env, splits.subList(mid + 1, splits.size())));
+
+ } catch (Exception e) {
+ env.exception.compareAndSet(null, e);
+ }
+ }
+
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param partitionKeys
+ * a sorted set of row key values to pre-split the table on
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public void addSplits(String tableName, SortedSet<Text> partitionKeys) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ String tableId = Tables.getTableId(instance, tableName);
+
+ List<Text> splits = new ArrayList<Text>(partitionKeys);
+ // should be sorted because we copied from a sorted set, but that makes assumptions about
+ // how the copy was done so resort to be sure.
+ Collections.sort(splits);
+
+ CountDownLatch latch = new CountDownLatch(splits.size());
+ AtomicReference<Exception> exception = new AtomicReference<Exception>(null);
+
+ ExecutorService executor = Executors.newFixedThreadPool(16, new NamingThreadFactory("addSplits"));
+ try {
+ executor.submit(new SplitTask(new SplitEnv(tableName, tableId, executor, latch, exception), splits));
+
+ while (!latch.await(100, TimeUnit.MILLISECONDS)) {
+ if (exception.get() != null) {
+ executor.shutdownNow();
+ Exception excep = exception.get();
+ if (excep instanceof TableNotFoundException)
+ throw (TableNotFoundException) excep;
+ else if (excep instanceof AccumuloException)
+ throw (AccumuloException) excep;
+ else if (excep instanceof AccumuloSecurityException)
+ throw (AccumuloSecurityException) excep;
+ else if (excep instanceof RuntimeException)
+ throw (RuntimeException) excep;
+ else
+ throw new RuntimeException(excep);
+ }
+ }
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ } finally {
+ executor.shutdown();
+ }
+ }
+
+ private void addSplits(String tableName, SortedSet<Text> partitionKeys, String tableId) throws AccumuloException, AccumuloSecurityException,
+ TableNotFoundException, AccumuloServerException {
+ TabletLocator tabLocator = TabletLocator.getInstance(instance, new Text(tableId));
+
+ for (Text split : partitionKeys) {
+ boolean successful = false;
+ int attempt = 0;
+
+ while (!successful) {
+
+ if (attempt > 0)
+ UtilWaitThread.sleep(100);
+
+ attempt++;
+
+ TabletLocation tl = tabLocator.locateTablet(split, false, false, credentials);
+
+ if (tl == null) {
+ if (!Tables.exists(instance, tableId))
+ throw new TableNotFoundException(tableId, tableName, null);
+ else if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
+ throw new TableOfflineException(instance, tableId);
+ continue;
+ }
+
+ try {
+ TabletClientService.Client client = ThriftUtil.getTServerClient(tl.tablet_location, instance.getConfiguration());
+ try {
+ OpTimer opTimer = null;
+ if (log.isTraceEnabled())
+ opTimer = new OpTimer(log, Level.TRACE).start("Splitting tablet " + tl.tablet_extent + " on " + tl.tablet_location + " at " + split);
+
+ client.splitTablet(Tracer.traceInfo(), credentials, tl.tablet_extent.toThrift(), TextUtil.getByteBuffer(split));
+
+ // just split it, might as well invalidate it in the cache
+ tabLocator.invalidateCache(tl.tablet_extent);
+
+ if (opTimer != null)
+ opTimer.stop("Split tablet in %DURATION%");
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+
+ } catch (TApplicationException tae) {
+ throw new AccumuloServerException(tl.tablet_location, tae);
+ } catch (TTransportException e) {
+ tabLocator.invalidateCache(tl.tablet_location);
+ continue;
+ } catch (ThriftSecurityException e) {
+ Tables.clearCache(instance);
+ if (!Tables.exists(instance, tableId))
+ throw new TableNotFoundException(tableId, tableName, null);
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (NotServingTabletException e) {
+ tabLocator.invalidateCache(tl.tablet_extent);
+ continue;
+ } catch (TException e) {
+ tabLocator.invalidateCache(tl.tablet_location);
+ continue;
+ }
+
+ successful = true;
+ }
+ }
+ }
+
+ @Override
+ public void merge(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+
+ ArgumentChecker.notNull(tableName);
+ ByteBuffer EMPTY = ByteBuffer.allocate(0);
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), start == null ? EMPTY : TextUtil.getByteBuffer(start), end == null ? EMPTY
+ : TextUtil.getByteBuffer(end));
+ Map<String,String> opts = new HashMap<String,String>();
+ try {
+ doTableOperation(TableOperation.MERGE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void deleteRows(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+
+ ArgumentChecker.notNull(tableName);
+ ByteBuffer EMPTY = ByteBuffer.allocate(0);
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), start == null ? EMPTY : TextUtil.getByteBuffer(start), end == null ? EMPTY
+ : TextUtil.getByteBuffer(end));
+ Map<String,String> opts = new HashMap<String,String>();
+ try {
+ doTableOperation(TableOperation.DELETE_RANGE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @return the split points (end-row names) for the table's current split profile
+ */
+ @Override
+ public Collection<Text> listSplits(String tableName) throws TableNotFoundException, AccumuloSecurityException {
+
+ ArgumentChecker.notNull(tableName);
+
+ String tableId = Tables.getTableId(instance, tableName);
+
+ SortedSet<KeyExtent> tablets = new TreeSet<KeyExtent>();
+ Map<KeyExtent,String> locations = new TreeMap<KeyExtent,String>();
+
+ while (true) {
+ try {
+ tablets.clear();
+ locations.clear();
+ // the following method throws AccumuloException for some conditions that should be retried
+ MetadataTable.getEntries(instance, credentials, tableId, true, locations, tablets);
+ break;
+ } catch (AccumuloSecurityException ase) {
+ throw ase;
+ } catch (Throwable t) {
+ if (!Tables.exists(instance, tableId)) {
+ throw new TableNotFoundException(tableId, tableName, null);
+ }
+
+ if (t instanceof RuntimeException && t.getCause() instanceof AccumuloSecurityException) {
+ throw (AccumuloSecurityException) t.getCause();
+ }
+
+ log.info(t.getMessage() + " ... retrying ...");
+ UtilWaitThread.sleep(3000);
+ }
+ }
+
+ ArrayList<Text> endRows = new ArrayList<Text>(tablets.size());
+
+ for (KeyExtent ke : tablets)
+ if (ke.getEndRow() != null)
+ endRows.add(ke.getEndRow());
+
+ return endRows;
+ }
+
+ @Deprecated
+ @Override
+ public Collection<Text> getSplits(String tableName) throws TableNotFoundException {
+ try {
+ return listSplits(tableName);
+ } catch (AccumuloSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param maxSplits
+ * specifies the maximum number of splits to return
+ * @return the split points (end-row names) for the table's current split profile, grouped into fewer splits so as not to exceed maxSplits
- * @throws TableNotFoundException
+ */
+ @Override
+ public Collection<Text> listSplits(String tableName, int maxSplits) throws TableNotFoundException, AccumuloSecurityException {
+ Collection<Text> endRows = listSplits(tableName);
+
+ if (endRows.size() <= maxSplits)
+ return endRows;
+
+ double r = (maxSplits + 1) / (double) (endRows.size());
+ double pos = 0;
+
+ ArrayList<Text> subset = new ArrayList<Text>(maxSplits);
+
+ int j = 0;
+ for (int i = 0; i < endRows.size() && j < maxSplits; i++) {
+ pos += r;
+ while (pos > 1) {
+ subset.add(((ArrayList<Text>) endRows).get(i));
+ j++;
+ pos -= 1;
+ }
+ }
+
+ return subset;
+ }
+
+ @Deprecated
+ @Override
+ public Collection<Text> getSplits(String tableName, int maxSplits) throws TableNotFoundException {
+ try {
+ return listSplits(tableName, maxSplits);
+ } catch (AccumuloSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Delete a table
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public void delete(String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)));
+ Map<String,String> opts = new HashMap<String,String>();
+
+ try {
+ doTableOperation(TableOperation.DELETE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ @Override
+ public void clone(String srcTableName, String newTableName, boolean flush, Map<String,String> propertiesToSet, Set<String> propertiesToExclude)
+ throws AccumuloSecurityException, TableNotFoundException, AccumuloException, TableExistsException {
+
+ ArgumentChecker.notNull(srcTableName, newTableName);
+
+ String srcTableId = Tables.getTableId(instance, srcTableName);
+
+ if (flush)
+ _flush(srcTableId, null, null, true);
+
+ if (propertiesToExclude == null)
+ propertiesToExclude = Collections.emptySet();
+
+ if (propertiesToSet == null)
+ propertiesToSet = Collections.emptyMap();
+
+ if (!Collections.disjoint(propertiesToExclude, propertiesToSet.keySet()))
+ throw new IllegalArgumentException("propertiesToSet and propertiesToExclude not disjoint");
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(srcTableId.getBytes(Constants.UTF8)), ByteBuffer.wrap(newTableName.getBytes(Constants.UTF8)));
+ Map<String,String> opts = new HashMap<String,String>();
+ for (Entry<String,String> entry : propertiesToSet.entrySet()) {
+ if (entry.getKey().startsWith(CLONE_EXCLUDE_PREFIX))
+ throw new IllegalArgumentException("Property can not start with " + CLONE_EXCLUDE_PREFIX);
+ opts.put(entry.getKey(), entry.getValue());
+ }
+
+ for (String prop : propertiesToExclude) {
+ opts.put(CLONE_EXCLUDE_PREFIX + prop, "");
+ }
+
+ doTableOperation(TableOperation.CLONE, args, opts);
+ }
+
+ /**
+ * Rename a table
+ *
+ * @param oldTableName
+ * the old table name
+ * @param newTableName
+ * the new table name
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the old table name does not exist
+ * @throws TableExistsException
+ * if the new table name already exists
+ */
+ @Override
+ public void rename(String oldTableName, String newTableName) throws AccumuloSecurityException, TableNotFoundException, AccumuloException,
+ TableExistsException {
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(oldTableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(newTableName.getBytes(Constants.UTF8)));
+ Map<String,String> opts = new HashMap<String,String>();
+ doTableOperation(TableOperation.RENAME, args, opts);
+ }
+
+ /**
+ * @deprecated since 1.4 {@link #flush(String, Text, Text, boolean)}
+ */
+ @Override
+ @Deprecated
+ public void flush(String tableName) throws AccumuloException, AccumuloSecurityException {
+ try {
+ flush(tableName, null, null, false);
+ } catch (TableNotFoundException e) {
+ throw new AccumuloException(e.getMessage(), e);
+ }
+ }
+
+ /**
+ * Flush a table
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
- * @throws TableNotFoundException
+ */
+ @Override
+ public void flush(String tableName, Text start, Text end, boolean wait) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+
+ String tableId = Tables.getTableId(instance, tableName);
+ _flush(tableId, start, end, wait);
+ }
+
+ @Override
+ public void compact(String tableName, Text start, Text end, boolean flush, boolean wait) throws AccumuloSecurityException, TableNotFoundException,
+ AccumuloException {
+ compact(tableName, start, end, new ArrayList<IteratorSetting>(), flush, wait);
+ }
+
+ @Override
+ public void compact(String tableName, Text start, Text end, List<IteratorSetting> iterators, boolean flush, boolean wait) throws AccumuloSecurityException,
+ TableNotFoundException, AccumuloException {
+ ArgumentChecker.notNull(tableName);
+ ByteBuffer EMPTY = ByteBuffer.allocate(0);
+
+ String tableId = Tables.getTableId(instance, tableName);
+
+ if (flush)
+ _flush(tableId, start, end, true);
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)), start == null ? EMPTY : TextUtil.getByteBuffer(start), end == null ? EMPTY
+ : TextUtil.getByteBuffer(end), ByteBuffer.wrap(IteratorUtil.encodeIteratorSettings(iterators)));
+
+ Map<String,String> opts = new HashMap<String,String>();
+ try {
+ doTableOperation(TableOperation.COMPACT, args, opts, wait);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void cancelCompaction(String tableName) throws AccumuloSecurityException, TableNotFoundException, AccumuloException {
+ String tableId = Tables.getTableId(instance, tableName);
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)));
+
+ Map<String,String> opts = new HashMap<String,String>();
+ try {
+ doTableOperation(TableOperation.COMPACT_CANCEL, args, opts, true);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ private void _flush(String tableId, Text start, Text end, boolean wait) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+
+ try {
+ long flushID;
+
+ // used to pass the table name. but the tableid associated with a table name could change between calls.
+ // so pass the tableid to both calls
+
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ flushID = client.initiateFlush(Tracer.traceInfo(), credentials, tableId);
+ break;
+ } catch (TTransportException tte) {
+ log.debug("Failed to call initiateFlush, retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ client.waitForFlush(Tracer.traceInfo(), credentials, tableId, TextUtil.getByteBuffer(start), TextUtil.getByteBuffer(end), flushID,
+ wait ? Long.MAX_VALUE : 1);
+ break;
+ } catch (TTransportException tte) {
+ log.debug("Failed to call initiateFlush, retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ } catch (ThriftSecurityException e) {
+ switch (e.getCode()) {
+ case TABLE_DOESNT_EXIST:
+ throw new TableNotFoundException(tableId, null, e.getMessage(), e);
+ default:
+ log.debug("flush security exception on table id " + tableId);
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ }
+ } catch (ThriftTableOperationException e) {
+ switch (e.getType()) {
+ case NOTFOUND:
+ throw new TableNotFoundException(e);
+ case OTHER:
+ default:
+ throw new AccumuloException(e.description, e);
+ }
+ } catch (Exception e) {
+ throw new AccumuloException(e);
+ }
+ }
+
+ /**
+ * Sets a property on a table
+ *
+ * @param tableName
+ * the name of the table
+ * @param property
+ * the name of a per-table property
+ * @param value
+ * the value to set a per-table property to
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ @Override
+ public void setProperty(final String tableName, final String property, final String value) throws AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, property, value);
+ MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() {
+ @Override
+ public void execute(MasterClientService.Client client) throws Exception {
+ client.setTableProperty(Tracer.traceInfo(), credentials, tableName, property, value);
+ }
+ });
+ }
+
+ /**
+ * Removes a property from a table
+ *
+ * @param tableName
+ * the name of the table
+ * @param property
+ * the name of a per-table property
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ @Override
+ public void removeProperty(final String tableName, final String property) throws AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, property);
+ MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() {
+ @Override
+ public void execute(MasterClientService.Client client) throws Exception {
+ client.removeTableProperty(Tracer.traceInfo(), credentials, tableName, property);
+ }
+ });
+ }
+
+ /**
+ * Gets properties of a table
+ *
+ * @param tableName
+ * the name of the table
+ * @return all properties visible by this table (system and per-table properties)
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public Iterable<Entry<String,String>> getProperties(final String tableName) throws AccumuloException, TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+ try {
+ return ServerClient.executeRaw(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() {
+ @Override
+ public Map<String,String> execute(ClientService.Client client) throws Exception {
+ return client.getTableConfiguration(Tracer.traceInfo(), credentials, tableName);
+ }
+ }).entrySet();
+ } catch (ThriftTableOperationException e) {
+ switch (e.getType()) {
+ case NOTFOUND:
+ throw new TableNotFoundException(e);
+ case OTHER:
+ default:
+ throw new AccumuloException(e.description, e);
+ }
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AccumuloException(e);
+ }
+
+ }
+
+ /**
+ * Sets a tables locality groups. A tables locality groups can be changed at any time.
+ *
+ * @param tableName
+ * the name of the table
+ * @param groups
+ * mapping of locality group names to column families in the locality group
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public void setLocalityGroups(String tableName, Map<String,Set<Text>> groups) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ // ensure locality groups do not overlap
+ HashSet<Text> all = new HashSet<Text>();
+ for (Entry<String,Set<Text>> entry : groups.entrySet()) {
+
+ if (!Collections.disjoint(all, entry.getValue())) {
+ throw new IllegalArgumentException("Group " + entry.getKey() + " overlaps with another group");
+ }
+
+ all.addAll(entry.getValue());
+ }
+
+ for (Entry<String,Set<Text>> entry : groups.entrySet()) {
+ Set<Text> colFams = entry.getValue();
+ String value = LocalityGroupUtil.encodeColumnFamilies(colFams);
+ setProperty(tableName, Property.TABLE_LOCALITY_GROUP_PREFIX + entry.getKey(), value);
+ }
+
+ setProperty(tableName, Property.TABLE_LOCALITY_GROUPS.getKey(), StringUtil.join(groups.keySet(), ","));
+
+ // remove anything extraneous
+ String prefix = Property.TABLE_LOCALITY_GROUP_PREFIX.getKey();
+ for (Entry<String,String> entry : getProperties(tableName)) {
+ String property = entry.getKey();
+ if (property.startsWith(prefix)) {
+ // this property configures a locality group, find out which
+ // one:
+ String[] parts = property.split("\\.");
+ String group = parts[parts.length - 1];
+
+ if (!groups.containsKey(group)) {
+ removeProperty(tableName, property);
+ }
+ }
+ }
+ }
+
+ /**
+ *
+ * Gets the locality groups currently set for a table.
+ *
+ * @param tableName
+ * the name of the table
+ * @return mapping of locality group names to column families in the locality group
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public Map<String,Set<Text>> getLocalityGroups(String tableName) throws AccumuloException, TableNotFoundException {
+ AccumuloConfiguration conf = new ConfigurationCopy(this.getProperties(tableName));
+ Map<String,Set<ByteSequence>> groups = LocalityGroupUtil.getLocalityGroups(conf);
+
+ Map<String,Set<Text>> groups2 = new HashMap<String,Set<Text>>();
+ for (Entry<String,Set<ByteSequence>> entry : groups.entrySet()) {
+
+ HashSet<Text> colFams = new HashSet<Text>();
+
+ for (ByteSequence bs : entry.getValue()) {
+ colFams.add(new Text(bs.toArray()));
+ }
+
+ groups2.put(entry.getKey(), colFams);
+ }
+
+ return groups2;
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param range
+ * a range to split
+ * @param maxSplits
+ * the maximum number of splits
+ * @return the range, split into smaller ranges that fall on boundaries of the table's split points as evenly as possible
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public Set<Range> splitRangeByTablets(String tableName, Range range, int maxSplits) throws AccumuloException, AccumuloSecurityException,
+ TableNotFoundException {
+ ArgumentChecker.notNull(tableName, range);
+ if (maxSplits < 1)
+ throw new IllegalArgumentException("maximum splits must be >= 1");
+ if (maxSplits == 1)
+ return Collections.singleton(range);
+
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+ String tableId = Tables.getTableId(instance, tableName);
+ TabletLocator tl = TabletLocator.getInstance(instance, new Text(tableId));
+ // its possible that the cache could contain complete, but old information about a tables tablets... so clear it
+ tl.invalidateCache();
+ while (!tl.binRanges(Collections.singletonList(range), binnedRanges, credentials).isEmpty()) {
+ if (!Tables.exists(instance, tableId))
+ throw new TableDeletedException(tableId);
+ if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
+ throw new TableOfflineException(instance, tableId);
+
+ log.warn("Unable to locate bins for specified range. Retrying.");
+ // sleep randomly between 100 and 200ms
+ UtilWaitThread.sleep(100 + (int) (Math.random() * 100));
+ binnedRanges.clear();
+ tl.invalidateCache();
+ }
+
+ // group key extents to get <= maxSplits
+ LinkedList<KeyExtent> unmergedExtents = new LinkedList<KeyExtent>();
+ List<KeyExtent> mergedExtents = new ArrayList<KeyExtent>();
+
+ for (Map<KeyExtent,List<Range>> map : binnedRanges.values())
+ unmergedExtents.addAll(map.keySet());
+
+ // the sort method is efficient for linked list
+ Collections.sort(unmergedExtents);
+
+ while (unmergedExtents.size() + mergedExtents.size() > maxSplits) {
+ if (unmergedExtents.size() >= 2) {
+ KeyExtent first = unmergedExtents.removeFirst();
+ KeyExtent second = unmergedExtents.removeFirst();
+ first.setEndRow(second.getEndRow());
+ mergedExtents.add(first);
+ } else {
+ mergedExtents.addAll(unmergedExtents);
+ unmergedExtents.clear();
+ unmergedExtents.addAll(mergedExtents);
+ mergedExtents.clear();
+ }
+
+ }
+
+ mergedExtents.addAll(unmergedExtents);
+
+ Set<Range> ranges = new HashSet<Range>();
+ for (KeyExtent k : mergedExtents)
+ ranges.add(k.toDataRange().clip(range));
+
+ return ranges;
+ }
+
+ @Override
+ public void importDirectory(String tableName, String dir, String failureDir, boolean setTime) throws IOException, AccumuloSecurityException,
+ TableNotFoundException, AccumuloException {
+ ArgumentChecker.notNull(tableName, dir, failureDir);
+ FileSystem fs = FileUtil.getFileSystem(CachedConfiguration.getInstance(), instance.getConfiguration());
+ Path dirPath = fs.makeQualified(new Path(dir));
+ Path failPath = fs.makeQualified(new Path(failureDir));
+ if (!fs.exists(dirPath))
+ throw new AccumuloException("Bulk import directory " + dir + " does not exist!");
+ if (!fs.exists(failPath))
+ throw new AccumuloException("Bulk import failure directory " + failureDir + " does not exist!");
+ FileStatus[] listStatus = fs.listStatus(failPath);
+ if (listStatus != null && listStatus.length != 0) {
+ if (listStatus.length == 1 && listStatus[0].isDir())
+ throw new AccumuloException("Bulk import directory " + failPath + " is a file");
+ throw new AccumuloException("Bulk import failure directory " + failPath + " is not empty");
+ }
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(dirPath.toString().getBytes(Constants.UTF8)),
+ ByteBuffer.wrap(failPath.toString().getBytes(Constants.UTF8)), ByteBuffer.wrap((setTime + "").getBytes(Constants.UTF8)));
+ Map<String,String> opts = new HashMap<String,String>();
+
+ try {
+ doTableOperation(TableOperation.BULK_IMPORT, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+ // return new BulkImportHelper(instance, credentials, tableName).importDirectory(new Path(dir), new Path(failureDir), numThreads, numAssignThreads,
+ // disableGC);
+ }
+
+ /**
+ *
+ * @param tableName
+ * the table to take offline
+ * @throws AccumuloException
+ * when there is a general accumulo error
+ * @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
- * @throws TableNotFoundException
+ */
+ @Override
+ public void offline(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
+
+ ArgumentChecker.notNull(tableName);
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)));
+ Map<String,String> opts = new HashMap<String,String>();
+
+ try {
+ doTableOperation(TableOperation.OFFLINE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ *
+ * @param tableName
+ * the table to take online
+ * @throws AccumuloException
+ * when there is a general accumulo error
+ * @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
- * @throws TableNotFoundException
+ */
+ @Override
+ public void online(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)));
+ Map<String,String> opts = new HashMap<String,String>();
+
+ try {
+ doTableOperation(TableOperation.ONLINE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Clears the tablet locator cache for a specified table
+ *
+ * @param tableName
+ * the name of the table
+ * @throws TableNotFoundException
+ * if table does not exist
+ */
+ @Override
+ public void clearLocatorCache(String tableName) throws TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+ TabletLocator tabLocator = TabletLocator.getInstance(instance, new Text(Tables.getTableId(instance, tableName)));
+ tabLocator.invalidateCache();
+ }
+
+ /**
+ * Get a mapping of table name to internal table id.
+ *
+ * @return the map from table name to internal table id
+ */
+ @Override
+ public Map<String,String> tableIdMap() {
+ return Tables.getNameToIdMap(instance);
+ }
+
+ @Override
+ public Text getMaxRow(String tableName, Authorizations auths, Text startRow, boolean startInclusive, Text endRow, boolean endInclusive)
+ throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, auths);
+ Scanner scanner = instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials)).createScanner(tableName, auths);
+ return FindMax.findMax(scanner, startRow, startInclusive, endRow, endInclusive);
+ }
+
+ public static Map<String,String> getExportedProps(FileSystem fs, Path path) throws IOException {
+ HashMap<String,String> props = new HashMap<String,String>();
+
+ ZipInputStream zis = new ZipInputStream(fs.open(path));
+ try {
+ ZipEntry zipEntry;
+ while ((zipEntry = zis.getNextEntry()) != null) {
+ if (zipEntry.getName().equals(Constants.EXPORT_TABLE_CONFIG_FILE)) {
+ BufferedReader in = new BufferedReader(new InputStreamReader(zis, Constants.UTF8));
+ String line;
+ while ((line = in.readLine()) != null) {
+ String sa[] = line.split("=", 2);
+ props.put(sa[0], sa[1]);
+ }
+
+ break;
+ }
+ }
+ } finally {
+ zis.close();
+ }
+ return props;
+ }
+
+ @Override
+ public void importTable(String tableName, String importDir) throws TableExistsException, AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, importDir);
+
+ try {
+ FileSystem fs = FileUtil.getFileSystem(CachedConfiguration.getInstance(), instance.getConfiguration());
+
+ Map<String,String> props = getExportedProps(fs, new Path(importDir, Constants.EXPORT_FILE));
+
+ for (Entry<String,String> prop : props.entrySet()) {
+ if (Property.isClassProperty(prop.getKey()) && !prop.getValue().contains(Constants.CORE_PACKAGE_NAME)) {
+ Logger.getLogger(this.getClass()).info(
+ "Imported table sets '" + prop.getKey() + "' to '" + prop.getValue() + "'. Ensure this class is on Accumulo classpath.");
+ }
+ }
+
+ } catch (IOException ioe) {
+ Logger.getLogger(this.getClass()).warn("Failed to check if imported table references external java classes : " + ioe.getMessage());
+ }
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(importDir.getBytes(Constants.UTF8)));
+
+ Map<String,String> opts = Collections.emptyMap();
+
+ try {
+ doTableOperation(TableOperation.IMPORT, args, opts);
+ } catch (TableNotFoundException e1) {
+ // should not happen
+ throw new RuntimeException(e1);
+ }
+
+ }
+
+ @Override
+ public void exportTable(String tableName, String exportDir) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, exportDir);
+
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(exportDir.getBytes(Constants.UTF8)));
+
+ Map<String,String> opts = Collections.emptyMap();
+
+ try {
+ doTableOperation(TableOperation.EXPORT, args, opts);
+ } catch (TableExistsException e1) {
+ // should not happen
+ throw new RuntimeException(e1);
+ }
+ }
+
+ @Override
+ public boolean testClassLoad(final String tableName, final String className, final String asTypeName) throws TableNotFoundException, AccumuloException,
+ AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, className, asTypeName);
+
+ try {
+ return ServerClient.executeRaw(instance, new ClientExecReturn<Boolean,ClientService.Client>() {
+ @Override
+ public Boolean execute(ClientService.Client client) throws Exception {
+ return client.checkTableClass(Tracer.traceInfo(), credentials, tableName, className, asTypeName);
+ }
+ });
+ } catch (ThriftTableOperationException e) {
+ switch (e.getType()) {
+ case NOTFOUND:
+ throw new TableNotFoundException(e);
+ case OTHER:
+ default:
+ throw new AccumuloException(e.description, e);
+ }
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AccumuloException(e);
+ }
+ }
+
+ @Override
+ public void attachIterator(String tableName, IteratorSetting setting, EnumSet<IteratorScope> scopes) throws AccumuloSecurityException, AccumuloException,
+ TableNotFoundException {
+ testClassLoad(tableName, setting.getIteratorClass(), SortedKeyValueIterator.class.getName());
+ super.attachIterator(tableName, setting, scopes);
+ }
+
+ @Override
+ public int addConstraint(String tableName, String constraintClassName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ testClassLoad(tableName, constraintClassName, Constraint.class.getName());
+ return super.addConstraint(tableName, constraintClassName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
index e00fcc8,0000000..1132e74
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
@@@ -1,414 -1,0 +1,408 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.impl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.RowIterator;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Column;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.KeyValue;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVIterator;
+import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
+import org.apache.accumulo.core.iterators.system.ColumnQualifierFilter;
+import org.apache.accumulo.core.iterators.system.DeletingIterator;
+import org.apache.accumulo.core.iterators.system.MultiIterator;
+import org.apache.accumulo.core.iterators.system.VisibilityFilter;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.LocalityGroupUtil;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.Text;
+
+class OfflineIterator implements Iterator<Entry<Key,Value>> {
+
+ static class OfflineIteratorEnvironment implements IteratorEnvironment {
+ @Override
+ public SortedKeyValueIterator<Key,Value> reserveMapFileReader(String mapFileName) throws IOException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public AccumuloConfiguration getConfig() {
+ return AccumuloConfiguration.getDefaultConfiguration();
+ }
+
+ @Override
+ public IteratorScope getIteratorScope() {
+ return IteratorScope.scan;
+ }
+
+ @Override
+ public boolean isFullMajorCompaction() {
+ return false;
+ }
+
+ private ArrayList<SortedKeyValueIterator<Key,Value>> topLevelIterators = new ArrayList<SortedKeyValueIterator<Key,Value>>();
+
+ @Override
+ public void registerSideChannel(SortedKeyValueIterator<Key,Value> iter) {
+ topLevelIterators.add(iter);
+ }
+
+ SortedKeyValueIterator<Key,Value> getTopLevelIterator(SortedKeyValueIterator<Key,Value> iter) {
+ if (topLevelIterators.isEmpty())
+ return iter;
+ ArrayList<SortedKeyValueIterator<Key,Value>> allIters = new ArrayList<SortedKeyValueIterator<Key,Value>>(topLevelIterators);
+ allIters.add(iter);
+ return new MultiIterator(allIters, false);
+ }
+ }
+
+ private SortedKeyValueIterator<Key,Value> iter;
+ private Range range;
+ private KeyExtent currentExtent;
+ private Connector conn;
+ private String tableId;
+ private Authorizations authorizations;
+ private Instance instance;
+ private ScannerOptions options;
+ private ArrayList<SortedKeyValueIterator<Key,Value>> readers;
+ private AccumuloConfiguration config;
+
- /**
- * @param instance
- * @param credentials
- * @param authorizations
- * @param table
- */
+ public OfflineIterator(ScannerOptions options, Instance instance, TCredentials credentials, Authorizations authorizations, Text table, Range range) {
+ this.options = new ScannerOptions(options);
+ this.instance = instance;
+ this.range = range;
+
+ if (this.options.fetchedColumns.size() > 0) {
+ this.range = range.bound(this.options.fetchedColumns.first(), this.options.fetchedColumns.last());
+ }
+
+ this.tableId = table.toString();
+ this.authorizations = authorizations;
+ this.readers = new ArrayList<SortedKeyValueIterator<Key,Value>>();
+
+ try {
+ conn = instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials));
+ config = new ConfigurationCopy(conn.instanceOperations().getSiteConfiguration());
+ nextTablet();
+
+ while (iter != null && !iter.hasTop())
+ nextTablet();
+
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return iter != null && iter.hasTop();
+ }
+
+ @Override
+ public Entry<Key,Value> next() {
+ try {
+ byte[] v = iter.getTopValue().get();
+ // copy just like tablet server does, do this before calling next
+ KeyValue ret = new KeyValue(new Key(iter.getTopKey()), Arrays.copyOf(v, v.length));
+
+ iter.next();
+
+ while (iter != null && !iter.hasTop())
+ nextTablet();
+
+ return ret;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * @throws TableNotFoundException
+ * @throws IOException
+ * @throws AccumuloException
+ *
+ */
+ private void nextTablet() throws TableNotFoundException, AccumuloException, IOException {
+
+ Range nextRange = null;
+
+ if (currentExtent == null) {
+ Text startRow;
+
+ if (range.getStartKey() != null)
+ startRow = range.getStartKey().getRow();
+ else
+ startRow = new Text();
+
+ nextRange = new Range(new KeyExtent(new Text(tableId), startRow, null).getMetadataEntry(), true, null, false);
+ } else {
+
+ if (currentExtent.getEndRow() == null) {
+ iter = null;
+ return;
+ }
+
+ if (range.afterEndKey(new Key(currentExtent.getEndRow()).followingKey(PartialKey.ROW))) {
+ iter = null;
+ return;
+ }
+
+ nextRange = new Range(currentExtent.getMetadataEntry(), false, null, false);
+ }
+
+ List<String> relFiles = new ArrayList<String>();
+
+ Pair<KeyExtent,String> eloc = getTabletFiles(nextRange, relFiles);
+
+ while (eloc.getSecond() != null) {
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ Tables.clearCache(instance);
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ throw new AccumuloException("Table is online " + tableId + " cannot scan tablet in offline mode " + eloc.getFirst());
+ }
+ }
+
+ UtilWaitThread.sleep(250);
+
+ eloc = getTabletFiles(nextRange, relFiles);
+ }
+
+ KeyExtent extent = eloc.getFirst();
+
+ if (!extent.getTableId().toString().equals(tableId)) {
+ throw new AccumuloException(" did not find tablets for table " + tableId + " " + extent);
+ }
+
+ if (currentExtent != null && !extent.isPreviousExtent(currentExtent))
+ throw new AccumuloException(" " + currentExtent + " is not previous extent " + extent);
+
+ String tablesDir = Constants.getTablesDir(config);
+ List<String> absFiles = new ArrayList<String>();
+ for (String relPath : relFiles) {
+ if (relPath.startsWith(".."))
+ absFiles.add(tablesDir + relPath.substring(2));
+ else
+ absFiles.add(tablesDir + "/" + tableId + relPath);
+ }
+
+ iter = createIterator(extent, absFiles);
+ iter.seek(range, LocalityGroupUtil.families(options.fetchedColumns), options.fetchedColumns.size() == 0 ? false : true);
+ currentExtent = extent;
+
+ }
+
+ private Pair<KeyExtent,String> getTabletFiles(Range nextRange, List<String> relFiles) throws TableNotFoundException {
+ Scanner scanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ scanner.setBatchSize(100);
+ scanner.setRange(nextRange);
+
+ RowIterator rowIter = new RowIterator(scanner);
+ Iterator<Entry<Key,Value>> row = rowIter.next();
+
+ KeyExtent extent = null;
+ String location = null;
+
+ while (row.hasNext()) {
+ Entry<Key,Value> entry = row.next();
+ Key key = entry.getKey();
+
+ if (key.getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
+ relFiles.add(key.getColumnQualifier().toString());
+ }
+
+ if (key.getColumnFamily().equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY)
+ || key.getColumnFamily().equals(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY)) {
+ location = entry.getValue().toString();
+ }
+
+ if (Constants.METADATA_PREV_ROW_COLUMN.hasColumns(key)) {
+ extent = new KeyExtent(key.getRow(), entry.getValue());
+ }
+
+ }
+ return new Pair<KeyExtent,String>(extent, location);
+ }
+
+ /**
+ * @param absFiles
+ * @return
+ * @throws AccumuloException
+ * @throws TableNotFoundException
+ * @throws IOException
+ */
+ private SortedKeyValueIterator<Key,Value> createIterator(KeyExtent extent, List<String> absFiles) throws TableNotFoundException, AccumuloException,
+ IOException {
+
+ // TODO share code w/ tablet - ACCUMULO-1303
+ AccumuloConfiguration acuTableConf = AccumuloConfiguration.getTableConfiguration(conn, tableId);
+
+ Configuration conf = CachedConfiguration.getInstance();
+
+ FileSystem fs = FileUtil.getFileSystem(conf, config);
+
+ for (SortedKeyValueIterator<Key,Value> reader : readers) {
+ ((FileSKVIterator) reader).close();
+ }
+
+ readers.clear();
+
+ // TODO need to close files - ACCUMULO-1303
+ for (String file : absFiles) {
+ FileSKVIterator reader = FileOperations.getInstance().openReader(file, false, fs, conf, acuTableConf, null, null);
+ readers.add(reader);
+ }
+
+ MultiIterator multiIter = new MultiIterator(readers, extent);
+
+ OfflineIteratorEnvironment iterEnv = new OfflineIteratorEnvironment();
+
+ DeletingIterator delIter = new DeletingIterator(multiIter, false);
+
+ ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(delIter);
+
+ ColumnQualifierFilter colFilter = new ColumnQualifierFilter(cfsi, new HashSet<Column>(options.fetchedColumns));
+
+ byte[] defaultSecurityLabel;
+
+ ColumnVisibility cv = new ColumnVisibility(acuTableConf.get(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY));
+ defaultSecurityLabel = cv.getExpression();
+
+ VisibilityFilter visFilter = new VisibilityFilter(colFilter, authorizations, defaultSecurityLabel);
+
+ return iterEnv.getTopLevelIterator(IteratorUtil.loadIterators(IteratorScope.scan, visFilter, extent, acuTableConf, options.serverSideIteratorList,
+ options.serverSideIteratorOptions, iterEnv, false));
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+}
+
+/**
+ *
+ */
+public class OfflineScanner extends ScannerOptions implements Scanner {
+
+ private int batchSize;
+ private int timeOut;
+ private Range range;
+
+ private Instance instance;
+ private TCredentials credentials;
+ private Authorizations authorizations;
+ private Text tableId;
+
+ public OfflineScanner(Instance instance, TCredentials credentials, String tableId, Authorizations authorizations) {
+ ArgumentChecker.notNull(instance, credentials, tableId, authorizations);
+ this.instance = instance;
+ this.credentials = credentials;
+ this.tableId = new Text(tableId);
+ this.range = new Range((Key) null, (Key) null);
+
+ this.authorizations = authorizations;
+
+ this.batchSize = Constants.SCAN_BATCH_SIZE;
+ this.timeOut = Integer.MAX_VALUE;
+ }
+
+ @Deprecated
+ @Override
+ public void setTimeOut(int timeOut) {
+ this.timeOut = timeOut;
+ }
+
+ @Deprecated
+ @Override
+ public int getTimeOut() {
+ return timeOut;
+ }
+
+ @Override
+ public void setRange(Range range) {
+ this.range = range;
+ }
+
+ @Override
+ public Range getRange() {
+ return range;
+ }
+
+ @Override
+ public void setBatchSize(int size) {
+ this.batchSize = size;
+ }
+
+ @Override
+ public int getBatchSize() {
+ return batchSize;
+ }
+
+ @Override
+ public void enableIsolation() {
+
+ }
+
+ @Override
+ public void disableIsolation() {
+
+ }
+
+ @Override
+ public Iterator<Entry<Key,Value>> iterator() {
+ return new OfflineIterator(this, instance, credentials, authorizations, tableId, range);
+ }
+
+}
[51/64] [abbrv] git commit: Merge branch '1.4.6-SNAPSHOT' into
1.5.2-SNAPSHOT
Posted by ct...@apache.org.
Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Conflicts:
core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java
core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java
core/src/main/java/org/apache/accumulo/core/iterators/FirstEntryInRowIterator.java
core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
server/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
server/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
server/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
src/core/src/main/java/org/apache/accumulo/core/file/map/MapFileUtil.java
src/server/src/main/java/org/apache/accumulo/server/util/DumpMapFile.java
src/server/src/main/java/org/apache/accumulo/server/util/DumpTabletsOnServer.java
test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/92613388
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/92613388
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/92613388
Branch: refs/heads/master
Commit: 92613388919b6fc138e829382cc1cbb6647faa31
Parents: 5363d78 c8e165a
Author: Christopher Tubbs <ct...@apache.org>
Authored: Wed Apr 9 13:14:57 2014 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Wed Apr 9 13:14:57 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/Constants.java | 1 -
.../core/client/ClientSideIteratorScanner.java | 2 -
.../apache/accumulo/core/client/Connector.java | 2 -
.../accumulo/core/client/IteratorSetting.java | 4 -
.../accumulo/core/client/RowIterator.java | 4 -
.../accumulo/core/client/ScannerBase.java | 2 +-
.../accumulo/core/client/ZooKeeperInstance.java | 2 -
.../core/client/admin/ActiveCompaction.java | 1 -
.../core/client/admin/InstanceOperations.java | 12 -
.../client/admin/InstanceOperationsImpl.java | 33 --
.../core/client/admin/TableOperations.java | 32 --
.../core/client/admin/TableOperationsImpl.java | 4 -
.../core/client/impl/OfflineScanner.java | 6 -
.../core/client/impl/ThriftTransportPool.java | 16 +-
.../client/mapred/AccumuloOutputFormat.java | 1 -
.../core/client/mapred/InputFormatBase.java | 1 -
.../client/mapreduce/AccumuloOutputFormat.java | 1 -
.../core/client/mapreduce/InputFormatBase.java | 1 -
.../mapreduce/lib/util/ConfiguratorBase.java | 1 -
.../core/client/mock/MockBatchDeleter.java | 4 -
.../client/mock/MockInstanceOperations.java | 43 ---
.../apache/accumulo/core/data/ColumnUpdate.java | 1 -
.../java/org/apache/accumulo/core/data/Key.java | 7 +-
.../apache/accumulo/core/data/KeyExtent.java | 6 +-
.../org/apache/accumulo/core/data/Range.java | 17 +-
.../accumulo/core/file/rfile/BlockIndex.java | 5 -
.../accumulo/core/file/rfile/bcfile/BCFile.java | 14 +-
.../core/file/rfile/bcfile/ByteArray.java | 2 -
.../accumulo/core/file/rfile/bcfile/Chunk.java | 2 -
.../accumulo/core/file/rfile/bcfile/TFile.java | 44 ---
.../core/file/rfile/bcfile/TFileDumper.java | 1 -
.../accumulo/core/file/rfile/bcfile/Utils.java | 11 -
.../core/iterators/TypedValueCombiner.java | 6 -
.../core/iterators/ValueFormatException.java | 6 -
.../core/iterators/system/MapFileIterator.java | 8 +-
.../core/iterators/user/GrepIterator.java | 3 -
.../iterators/user/IntersectingIterator.java | 10 -
.../accumulo/core/iterators/user/RowFilter.java | 1 -
.../iterators/user/TransformingIterator.java | 166 +++++-----
.../core/iterators/user/VersioningIterator.java | 3 -
.../accumulo/core/security/SecurityUtil.java | 1 -
.../core/security/crypto/CryptoModule.java | 2 -
.../security/crypto/CryptoModuleFactory.java | 1 -
.../core/client/impl/ScannerOptionsTest.java | 2 -
.../client/mapred/AccumuloInputFormatTest.java | 4 -
.../mapreduce/AccumuloInputFormatTest.java | 6 -
.../shell/command/FormatterCommandTest.java | 15 +-
.../simple/client/RandomBatchScanner.java | 5 -
.../simple/client/RandomBatchWriter.java | 4 -
.../simple/client/SequentialBatchWriter.java | 5 -
.../simple/client/TraceDumpExample.java | 9 +-
.../examples/simple/dirlist/QueryUtil.java | 3 -
.../examples/simple/mapreduce/NGramIngest.java | 3 -
.../examples/simple/mapreduce/TableToFile.java | 1 -
.../accumulo/examples/simple/shard/Query.java | 3 -
.../minicluster/MiniAccumuloCluster.java | 10 -
.../accumulo/proxy/TestProxyReadWrite.java | 10 -
.../accumulo/server/conf/ConfigSanityCheck.java | 3 -
.../accumulo/server/logger/LogReader.java | 1 -
.../master/balancer/ChaoticLoadBalancer.java | 8 -
.../server/master/balancer/TabletBalancer.java | 2 -
.../server/master/state/TabletStateStore.java | 8 +-
.../server/master/tableOps/TraceRepo.java | 25 --
.../server/metanalysis/LogFileOutputFormat.java | 4 -
.../server/metanalysis/PrintEvents.java | 5 +-
.../server/metrics/AbstractMetricsImpl.java | 4 -
.../security/handler/InsecurePermHandler.java | 42 ---
.../server/security/handler/ZKAuthorizor.java | 7 +-
.../server/security/handler/ZKPermHandler.java | 6 +-
.../accumulo/server/tabletserver/MemValue.java | 4 +-
.../server/tabletserver/TabletServer.java | 8 +-
.../server/util/AddFilesWithMissingEntries.java | 1 -
.../accumulo/server/util/DumpZookeeper.java | 3 -
.../server/util/FindOfflineTablets.java | 3 -
.../accumulo/server/util/LoginProperties.java | 3 -
.../accumulo/server/util/MetadataTable.java | 305 +++++++++----------
.../accumulo/server/util/RestoreZookeeper.java | 4 -
.../accumulo/server/util/SendLogToChainsaw.java | 1 -
.../accumulo/server/util/TServerUtils.java | 7 +-
.../accumulo/server/util/TableDiskUsage.java | 3 -
.../accumulo/server/util/TabletServerLocks.java | 3 -
.../start/classloader/AccumuloClassLoader.java | 3 -
.../start/classloader/vfs/ContextManager.java | 2 -
.../vfs/PostDelegatingVFSClassLoader.java | 7 +-
.../vfs/providers/HdfsFileSystem.java | 5 -
.../apache/accumulo/test/GetMasterStats.java | 9 -
.../accumulo/test/NativeMapPerformanceTest.java | 3 -
.../accumulo/test/NativeMapStressTest.java | 3 -
.../test/continuous/ContinuousMoru.java | 1 -
.../test/continuous/ContinuousVerify.java | 1 -
.../test/functional/CacheTestClean.java | 3 -
.../accumulo/test/functional/RunTests.java | 4 -
.../metadata/MetadataBatchScanTest.java | 3 -
.../test/performance/thrift/NullTserver.java | 8 +-
.../accumulo/test/randomwalk/Framework.java | 3 -
.../apache/accumulo/test/randomwalk/Node.java | 1 -
.../randomwalk/concurrent/CheckBalance.java | 5 +-
.../instrument/receivers/ZooSpanClient.java | 10 -
98 files changed, 284 insertions(+), 817 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/Constants.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/Constants.java
index 095319e,0000000..66c4034
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/Constants.java
+++ b/core/src/main/java/org/apache/accumulo/core/Constants.java
@@@ -1,213 -1,0 +1,212 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core;
+
+import java.nio.charset.Charset;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+
+public class Constants {
+ public static final Charset UTF8 = Charset.forName("UTF-8");
+ public static final String VERSION = FilteredConstants.VERSION;
+
+ // versions should never be negative
+ public static final Integer WIRE_VERSION = 2;
+ public static final int DATA_VERSION = 5;
+ public static final int PREV_DATA_VERSION = 4;
+
+ // Zookeeper locations
+ public static final String ZROOT = "/accumulo";
+ public static final String ZINSTANCES = "/instances";
+
+ public static final String ZTABLES = "/tables";
+ public static final byte[] ZTABLES_INITIAL_ID = new byte[] {'0'};
+ public static final String ZTABLE_NAME = "/name";
+ public static final String ZTABLE_CONF = "/conf";
+ public static final String ZTABLE_STATE = "/state";
+ public static final String ZTABLE_FLUSH_ID = "/flush-id";
+ public static final String ZTABLE_COMPACT_ID = "/compact-id";
+ public static final String ZTABLE_COMPACT_CANCEL_ID = "/compact-cancel-id";
+
+ public static final String ZROOT_TABLET = "/root_tablet";
+ public static final String ZROOT_TABLET_LOCATION = ZROOT_TABLET + "/location";
+ public static final String ZROOT_TABLET_FUTURE_LOCATION = ZROOT_TABLET + "/future_location";
+ public static final String ZROOT_TABLET_LAST_LOCATION = ZROOT_TABLET + "/lastlocation";
+ public static final String ZROOT_TABLET_WALOGS = ZROOT_TABLET + "/walogs";
+
+ public static final String ZMASTERS = "/masters";
+ public static final String ZMASTER_LOCK = ZMASTERS + "/lock";
+ public static final String ZMASTER_GOAL_STATE = ZMASTERS + "/goal_state";
+ public static final String ZGC = "/gc";
+ public static final String ZGC_LOCK = ZGC + "/lock";
+
+ public static final String ZMONITOR = "/monitor";
+ public static final String ZMONITOR_LOCK = ZMONITOR + "/lock";
+ public static final String ZMONITOR_HTTP_ADDR = ZMONITOR + "/http_addr";
+ public static final String ZMONITOR_LOG4J_ADDR = ZMONITOR + "/log4j_addr";
+
+ public static final String ZCONFIG = "/config";
+
+ public static final String ZTSERVERS = "/tservers";
+
+ public static final String ZDEAD = "/dead";
+ public static final String ZDEADTSERVERS = "/dead/tservers";
+
+ public static final String ZTRACERS = "/tracers";
+
+ public static final String ZPROBLEMS = "/problems";
+ public static final String ZUSERS = "/users";
+
+ public static final String BULK_ARBITRATOR_TYPE = "bulkTx";
+
+ public static final String ZFATE = "/fate";
+
+ public static final String ZNEXT_FILE = "/next_file";
+
+ public static final String ZBULK_FAILED_COPYQ = "/bulk_failed_copyq";
+
+ public static final String ZHDFS_RESERVATIONS = "/hdfs_reservations";
+ public static final String ZRECOVERY = "/recovery";
+
+ public static final String METADATA_TABLE_ID = "!0";
+ public static final String METADATA_TABLE_NAME = "!METADATA";
+ public static final String DEFAULT_TABLET_LOCATION = "/default_tablet";
+ public static final String TABLE_TABLET_LOCATION = "/table_info";
+ public static final String ZTABLE_LOCKS = "/table_locks";
+
+ // reserved keyspace is any row that begins with a tilde '~' character
+ public static final Key METADATA_RESERVED_KEYSPACE_START_KEY = new Key(new Text(new byte[] {'~'}));
+ public static final Key METADATA_RESERVED_KEYSPACE_STOP_KEY = new Key(new Text(new byte[] {'~' + 1}));
+ public static final Range METADATA_RESERVED_KEYSPACE = new Range(METADATA_RESERVED_KEYSPACE_START_KEY, true, METADATA_RESERVED_KEYSPACE_STOP_KEY, false);
+ public static final String METADATA_DELETE_FLAG_PREFIX = "~del";
+ public static final String METADATA_DELETE_FLAG_FOR_METADATA_PREFIX = "!!" + METADATA_DELETE_FLAG_PREFIX;
+ public static final Range METADATA_DELETES_KEYSPACE = new Range(new Key(new Text(METADATA_DELETE_FLAG_PREFIX)), true, new Key(new Text("~dem")), false);
+ public static final Range METADATA_DELETES_FOR_METADATA_KEYSPACE = new Range(new Key(new Text(METADATA_DELETE_FLAG_FOR_METADATA_PREFIX)), true, new Key(new Text("!!~dem")), false);
+ public static final String METADATA_BLIP_FLAG_PREFIX = "~blip"; // BLIP = bulk load in progress
+ public static final Range METADATA_BLIP_KEYSPACE = new Range(new Key(new Text(METADATA_BLIP_FLAG_PREFIX)), true, new Key(new Text("~bliq")), false);
+
+ public static final Text METADATA_SERVER_COLUMN_FAMILY = new Text("srv");
+ public static final Text METADATA_TABLET_COLUMN_FAMILY = new Text("~tab"); // this needs to sort after all other column families for that tablet
+ public static final Text METADATA_CURRENT_LOCATION_COLUMN_FAMILY = new Text("loc");
+ public static final Text METADATA_FUTURE_LOCATION_COLUMN_FAMILY = new Text("future");
+ public static final Text METADATA_LAST_LOCATION_COLUMN_FAMILY = new Text("last");
+ public static final Text METADATA_BULKFILE_COLUMN_FAMILY = new Text("loaded"); // temporary marker that indicates a tablet loaded a bulk file
+ public static final Text METADATA_CLONED_COLUMN_FAMILY = new Text("!cloned"); // temporary marker that indicates a tablet was successfully cloned
+
+ // README : very important that prevRow sort last to avoid race conditions between
+ // garbage collector and split
+ public static final ColumnFQ METADATA_PREV_ROW_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("~pr")); // this needs to sort after everything
+ // else for that tablet
+ public static final ColumnFQ METADATA_OLD_PREV_ROW_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("oldprevrow"));
+ public static final ColumnFQ METADATA_DIRECTORY_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("dir"));
+ public static final ColumnFQ METADATA_TIME_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("time"));
+ public static final ColumnFQ METADATA_FLUSH_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("flush"));
+ public static final ColumnFQ METADATA_COMPACT_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("compact"));
+ public static final ColumnFQ METADATA_SPLIT_RATIO_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("splitRatio"));
+ public static final ColumnFQ METADATA_LOCK_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("lock"));
+
+ public static final Text METADATA_DATAFILE_COLUMN_FAMILY = new Text("file");
+ public static final Text METADATA_SCANFILE_COLUMN_FAMILY = new Text("scan");
+ public static final Text METADATA_LOG_COLUMN_FAMILY = new Text("log");
+ public static final Text METADATA_CHOPPED_COLUMN_FAMILY = new Text("chopped");
+ public static final ColumnFQ METADATA_CHOPPED_COLUMN = new ColumnFQ(METADATA_CHOPPED_COLUMN_FAMILY, new Text("chopped"));
+
+ public static final Range NON_ROOT_METADATA_KEYSPACE = new Range(
+ new Key(KeyExtent.getMetadataEntry(new Text(METADATA_TABLE_ID), null)).followingKey(PartialKey.ROW), true, METADATA_RESERVED_KEYSPACE_START_KEY, false);
+ public static final Range METADATA_KEYSPACE = new Range(new Key(new Text(METADATA_TABLE_ID)), true, METADATA_RESERVED_KEYSPACE_START_KEY, false);
+
+ public static final KeyExtent ROOT_TABLET_EXTENT = new KeyExtent(new Text(METADATA_TABLE_ID), KeyExtent.getMetadataEntry(new Text(METADATA_TABLE_ID), null),
+ null);
+ public static final Range METADATA_ROOT_TABLET_KEYSPACE = new Range(ROOT_TABLET_EXTENT.getMetadataEntry(), false, KeyExtent.getMetadataEntry(new Text(
+ METADATA_TABLE_ID), null), true);
+
+ public static final String VALUE_ENCODING = "UTF-8";
+
+ public static final String BULK_PREFIX = "b-";
+ public static final String OLD_BULK_PREFIX = "bulk_";
+
+ // note: all times are in milliseconds
+
+ public static final int SCAN_BATCH_SIZE = 1000; // this affects the table client caching of metadata
+
+ public static final long MIN_MASTER_LOOP_TIME = 1000;
+ public static final int MASTER_TABLETSERVER_CONNECTION_TIMEOUT = 3000;
+ public static final long CLIENT_SLEEP_BEFORE_RECONNECT = 1000;
+
+ // Security configuration
+ public static final String PW_HASH_ALGORITHM = "SHA-256";
+
+ // Representation of an empty set of authorizations
+ // (used throughout the code, because scans of metadata table and many tests do not set record-level visibility)
+ public static final Authorizations NO_AUTHS = new Authorizations();
+
+ public static final int DEFAULT_MINOR_COMPACTION_MAX_SLEEP_TIME = 60 * 3; // in seconds
+
+ public static final int MAX_DATA_TO_PRINT = 64;
+ public static final int CLIENT_RETRIES = 5;
+ public static final int TSERV_MINC_MAXCONCURRENT_NUMWAITING_MULTIPLIER = 2;
+ public static final String CORE_PACKAGE_NAME = "org.apache.accumulo.core";
+ public static final String OLD_PACKAGE_NAME = "cloudbase";
+ public static final String VALID_TABLE_NAME_REGEX = "^\\w+$";
+ public static final String MAPFILE_EXTENSION = "map";
+ public static final String GENERATED_TABLET_DIRECTORY_PREFIX = "t-";
+
+ public static final String EXPORT_METADATA_FILE = "metadata.bin";
+ public static final String EXPORT_TABLE_CONFIG_FILE = "table_config.txt";
+ public static final String EXPORT_FILE = "exportMetadata.zip";
+ public static final String EXPORT_INFO_FILE = "accumulo_export_info.txt";
+
+ public static String getBaseDir(final AccumuloConfiguration conf) {
+ return conf.get(Property.INSTANCE_DFS_DIR);
+ }
+
+ public static String getTablesDir(final AccumuloConfiguration conf) {
+ return getBaseDir(conf) + "/tables";
+ }
+
+ public static String getRecoveryDir(final AccumuloConfiguration conf) {
+ return getBaseDir(conf) + "/recovery";
+ }
+
+ public static Path getDataVersionLocation(final AccumuloConfiguration conf) {
+ return new Path(getBaseDir(conf) + "/version");
+ }
+
+ public static String getMetadataTableDir(final AccumuloConfiguration conf) {
+ return getTablesDir(conf) + "/" + METADATA_TABLE_ID;
+ }
+
+ public static String getRootTabletDir(final AccumuloConfiguration conf) {
+ return getMetadataTableDir(conf) + ZROOT_TABLET;
+ }
+
+ /**
- * @param conf
+ * @return The write-ahead log directory.
+ */
+ public static String getWalDirectory(final AccumuloConfiguration conf) {
+ return getBaseDir(conf) + "/wal";
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
index 3085f56,0000000..fbf8670
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
@@@ -1,257 -1,0 +1,255 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.impl.ScannerOptions;
+import org.apache.accumulo.core.client.mock.IteratorAdapter;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Column;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.hadoop.io.Text;
+
+/**
+ * A scanner that instantiates iterators on the client side instead of on the tablet server. This can be useful for testing iterators or in cases where you
+ * don't want iterators affecting the performance of tablet servers.<br>
+ * <br>
+ * Suggested usage:<br>
+ * <code>Scanner scanner = new ClientSideIteratorScanner(connector.createScanner(tableName, authorizations));</code><br>
+ * <br>
+ * Iterators added to this scanner will be run in the client JVM. Separate scan iterators can be run on the server side and client side by adding iterators to
+ * the source scanner (which will execute server side) and to the client side scanner (which will execute client side).
+ */
+public class ClientSideIteratorScanner extends ScannerOptions implements Scanner {
+ private int size;
+
+ private Range range;
+ private boolean isolated = false;
+
+ /**
+ * A class that wraps a Scanner in a SortedKeyValueIterator so that other accumulo iterators can use it as a source.
+ */
+ public class ScannerTranslator implements SortedKeyValueIterator<Key,Value> {
+ protected Scanner scanner;
+ Iterator<Entry<Key,Value>> iter;
+ Entry<Key,Value> top = null;
+
+ /**
+ * Constructs an accumulo iterator from a scanner.
+ *
+ * @param scanner
+ * the scanner to iterate over
+ */
+ public ScannerTranslator(final Scanner scanner) {
+ this.scanner = scanner;
+ }
+
+ @Override
+ public void init(final SortedKeyValueIterator<Key,Value> source, final Map<String,String> options, final IteratorEnvironment env) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean hasTop() {
+ return top != null;
+ }
+
+ @Override
+ public void next() throws IOException {
+ if (iter.hasNext())
+ top = iter.next();
+ else
+ top = null;
+ }
+
+ @Override
+ public void seek(final Range range, final Collection<ByteSequence> columnFamilies, final boolean inclusive) throws IOException {
+ if (!inclusive && columnFamilies.size() > 0) {
+ throw new IllegalArgumentException();
+ }
+ scanner.setRange(range);
+ scanner.clearColumns();
+ for (ByteSequence colf : columnFamilies) {
+ scanner.fetchColumnFamily(new Text(colf.toArray()));
+ }
+ iter = scanner.iterator();
+ next();
+ }
+
+ @Override
+ public Key getTopKey() {
+ return top.getKey();
+ }
+
+ @Override
+ public Value getTopValue() {
+ return top.getValue();
+ }
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(final IteratorEnvironment env) {
+ return new ScannerTranslator(scanner);
+ }
+ }
+
+ private ScannerTranslator smi;
+
+ /**
+ * Constructs a scanner that can execute client-side iterators.
+ *
+ * @param scanner
+ * the source scanner
+ */
+ public ClientSideIteratorScanner(final Scanner scanner) {
+ smi = new ScannerTranslator(scanner);
+ this.range = scanner.getRange();
+ this.size = scanner.getBatchSize();
+ this.timeOut = scanner.getTimeout(TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Sets the source Scanner.
- *
- * @param scanner
+ */
+ public void setSource(final Scanner scanner) {
+ smi = new ScannerTranslator(scanner);
+ }
+
+ @Override
+ public Iterator<Entry<Key,Value>> iterator() {
+ smi.scanner.setBatchSize(size);
+ smi.scanner.setTimeout(timeOut, TimeUnit.MILLISECONDS);
+ if (isolated)
+ smi.scanner.enableIsolation();
+ else
+ smi.scanner.disableIsolation();
+
+ final TreeMap<Integer,IterInfo> tm = new TreeMap<Integer,IterInfo>();
+
+ for (IterInfo iterInfo : serverSideIteratorList) {
+ tm.put(iterInfo.getPriority(), iterInfo);
+ }
+
+ SortedKeyValueIterator<Key,Value> skvi;
+ try {
+ skvi = IteratorUtil.loadIterators(smi, tm.values(), serverSideIteratorOptions, new IteratorEnvironment() {
+ @Override
+ public SortedKeyValueIterator<Key,Value> reserveMapFileReader(final String mapFileName) throws IOException {
+ return null;
+ }
+
+ @Override
+ public AccumuloConfiguration getConfig() {
+ return null;
+ }
+
+ @Override
+ public IteratorScope getIteratorScope() {
+ return null;
+ }
+
+ @Override
+ public boolean isFullMajorCompaction() {
+ return false;
+ }
+
+ @Override
+ public void registerSideChannel(final SortedKeyValueIterator<Key,Value> iter) {}
+ }, false, null);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ final Set<ByteSequence> colfs = new TreeSet<ByteSequence>();
+ for (Column c : this.getFetchedColumns()) {
+ colfs.add(new ArrayByteSequence(c.getColumnFamily()));
+ }
+
+ try {
+ skvi.seek(range, colfs, true);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ return new IteratorAdapter(skvi);
+ }
+
+ @Deprecated
+ @Override
+ public void setTimeOut(int timeOut) {
+ if (timeOut == Integer.MAX_VALUE)
+ setTimeout(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
+ else
+ setTimeout(timeOut, TimeUnit.SECONDS);
+ }
+
+ @Deprecated
+ @Override
+ public int getTimeOut() {
+ long timeout = getTimeout(TimeUnit.SECONDS);
+ if (timeout >= Integer.MAX_VALUE)
+ return Integer.MAX_VALUE;
+ return (int) timeout;
+ }
+
+ @Override
+ public void setRange(final Range range) {
+ this.range = range;
+ }
+
+ @Override
+ public Range getRange() {
+ return range;
+ }
+
+ @Override
+ public void setBatchSize(final int size) {
+ this.size = size;
+ }
+
+ @Override
+ public int getBatchSize() {
+ return size;
+ }
+
+ @Override
+ public void enableIsolation() {
+ this.isolated = true;
+ }
+
+ @Override
+ public void disableIsolation() {
+ this.isolated = false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/Connector.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/Connector.java
index d2e7321,0000000..3189d44
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/Connector.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/Connector.java
@@@ -1,210 -1,0 +1,208 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
+import org.apache.accumulo.core.client.admin.InstanceOperations;
+import org.apache.accumulo.core.client.admin.SecurityOperations;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.security.Authorizations;
+
+/**
+ * Connector connects to an Accumulo instance and allows the user to request readers and writers for the instance as well as various objects that permit
+ * administrative operations.
+ *
+ * The Connector enforces security on the client side by forcing all API calls to be accompanied by user credentials.
+ */
+public abstract class Connector {
+
+ /**
+ * Factory method to create a BatchScanner connected to Accumulo.
+ *
+ * @param tableName
+ * the name of the table to query
+ * @param authorizations
+ * A set of authorization labels that will be checked against the column visibility of each key in order to filter data. The authorizations passed in
+ * must be a subset of the accumulo user's set of authorizations. If the accumulo user has authorizations (A1, A2) and authorizations (A2, A3) are
+ * passed, then an exception will be thrown.
+ * @param numQueryThreads
+ * the number of concurrent threads to spawn for querying
+ *
+ * @return BatchScanner object for configuring and querying
+ * @throws TableNotFoundException
+ * when the specified table doesn't exist
+ */
+ public abstract BatchScanner createBatchScanner(String tableName, Authorizations authorizations, int numQueryThreads) throws TableNotFoundException;
+
+ /**
+ * Factory method to create a BatchDeleter connected to Accumulo.
+ *
+ * @param tableName
+ * the name of the table to query and delete from
+ * @param authorizations
+ * A set of authorization labels that will be checked against the column visibility of each key in order to filter data. The authorizations passed in
+ * must be a subset of the accumulo user's set of authorizations. If the accumulo user has authorizations (A1, A2) and authorizations (A2, A3) are
+ * passed, then an exception will be thrown.
+ * @param numQueryThreads
+ * the number of concurrent threads to spawn for querying
+ * @param maxMemory
+ * size in bytes of the maximum memory to batch before writing
+ * @param maxLatency
+ * size in milliseconds; set to 0 or Long.MAX_VALUE to allow the maximum time to hold a batch before writing
+ * @param maxWriteThreads
+ * the maximum number of threads to use for writing data to the tablet servers
+ *
+ * @return BatchDeleter object for configuring and deleting
+ * @throws TableNotFoundException
+ * when the specified table doesn't exist
+ * @deprecated since 1.5.0; Use {@link #createBatchDeleter(String, Authorizations, int, BatchWriterConfig)} instead.
+ */
+ @Deprecated
+ public abstract BatchDeleter createBatchDeleter(String tableName, Authorizations authorizations, int numQueryThreads, long maxMemory, long maxLatency,
+ int maxWriteThreads) throws TableNotFoundException;
+
+ /**
+ *
+ * @param tableName
+ * the name of the table to query and delete from
+ * @param authorizations
+ * A set of authorization labels that will be checked against the column visibility of each key in order to filter data. The authorizations passed in
+ * must be a subset of the accumulo user's set of authorizations. If the accumulo user has authorizations (A1, A2) and authorizations (A2, A3) are
+ * passed, then an exception will be thrown.
+ * @param numQueryThreads
+ * the number of concurrent threads to spawn for querying
+ * @param config
+ * configuration used to create batch writer
+ * @return BatchDeleter object for configuring and deleting
- * @throws TableNotFoundException
+ * @since 1.5.0
+ */
+
+ public abstract BatchDeleter createBatchDeleter(String tableName, Authorizations authorizations, int numQueryThreads, BatchWriterConfig config)
+ throws TableNotFoundException;
+
+ /**
+ * Factory method to create a BatchWriter connected to Accumulo.
+ *
+ * @param tableName
+ * the name of the table to insert data into
+ * @param maxMemory
+ * size in bytes of the maximum memory to batch before writing
+ * @param maxLatency
+ * time in milliseconds; set to 0 or Long.MAX_VALUE to allow the maximum time to hold a batch before writing
+ * @param maxWriteThreads
+ * the maximum number of threads to use for writing data to the tablet servers
+ *
+ * @return BatchWriter object for configuring and writing data to
+ * @throws TableNotFoundException
+ * when the specified table doesn't exist
+ * @deprecated since 1.5.0; Use {@link #createBatchWriter(String, BatchWriterConfig)} instead.
+ */
+ @Deprecated
+ public abstract BatchWriter createBatchWriter(String tableName, long maxMemory, long maxLatency, int maxWriteThreads) throws TableNotFoundException;
+
+ /**
+ * Factory method to create a BatchWriter connected to Accumulo.
+ *
+ * @param tableName
+ * the name of the table to insert data into
+ * @param config
+ * configuration used to create batch writer
+ * @return BatchWriter object for configuring and writing data to
- * @throws TableNotFoundException
+ * @since 1.5.0
+ */
+
+ public abstract BatchWriter createBatchWriter(String tableName, BatchWriterConfig config) throws TableNotFoundException;
+
+ /**
+ * Factory method to create a Multi-Table BatchWriter connected to Accumulo. Multi-table batch writers can queue data for multiple tables, which is good for
+ * ingesting data into multiple tables from the same source
+ *
+ * @param maxMemory
+ * size in bytes of the maximum memory to batch before writing
+ * @param maxLatency
+ * size in milliseconds; set to 0 or Long.MAX_VALUE to allow the maximum time to hold a batch before writing
+ * @param maxWriteThreads
+ * the maximum number of threads to use for writing data to the tablet servers
+ *
+ * @return MultiTableBatchWriter object for configuring and writing data to
+ * @deprecated since 1.5.0; Use {@link #createMultiTableBatchWriter(BatchWriterConfig)} instead.
+ */
+ @Deprecated
+ public abstract MultiTableBatchWriter createMultiTableBatchWriter(long maxMemory, long maxLatency, int maxWriteThreads);
+
+ /**
+ * Factory method to create a Multi-Table BatchWriter connected to Accumulo. Multi-table batch writers can queue data for multiple tables. Also data for
+ * multiple tables can be sent to a server in a single batch. Its an efficient way to ingest data into multiple tables from a single process.
+ *
+ * @param config
+ * configuration used to create multi-table batch writer
+ * @return MultiTableBatchWriter object for configuring and writing data to
+ * @since 1.5.0
+ */
+
+ public abstract MultiTableBatchWriter createMultiTableBatchWriter(BatchWriterConfig config);
+
+ /**
+ * Factory method to create a Scanner connected to Accumulo.
+ *
+ * @param tableName
+ * the name of the table to query data from
+ * @param authorizations
+ * A set of authorization labels that will be checked against the column visibility of each key in order to filter data. The authorizations passed in
+ * must be a subset of the accumulo user's set of authorizations. If the accumulo user has authorizations (A1, A2) and authorizations (A2, A3) are
+ * passed, then an exception will be thrown.
+ *
+ * @return Scanner object for configuring and querying data with
+ * @throws TableNotFoundException
+ * when the specified table doesn't exist
+ */
+ public abstract Scanner createScanner(String tableName, Authorizations authorizations) throws TableNotFoundException;
+
+ /**
+ * Accessor method for internal instance object.
+ *
+ * @return the internal instance object
+ */
+ public abstract Instance getInstance();
+
+ /**
+ * Get the current user for this connector
+ *
+ * @return the user name
+ */
+ public abstract String whoami();
+
+ /**
+ * Retrieves a TableOperations object to perform table functions, such as create and delete.
+ *
+ * @return an object to manipulate tables
+ */
+ public abstract TableOperations tableOperations();
+
+ /**
+ * Retrieves a SecurityOperations object to perform user security operations, such as creating users.
+ *
+ * @return an object to modify users and permissions
+ */
+ public abstract SecurityOperations securityOperations();
+
+ /**
+ * Retrieves an InstanceOperations object to modify instance configuration.
+ *
+ * @return an object to modify instance configuration
+ */
+ public abstract InstanceOperations instanceOperations();
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
index 85e996a,0000000..e58a1be
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
@@@ -1,387 -1,0 +1,383 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+
+/**
+ * Configure an iterator for minc, majc, and/or scan. By default, IteratorSetting will be configured for scan.
+ *
+ * Every iterator has a priority, a name, a class, a set of scopes, and configuration parameters.
+ *
+ * A typical use case configured for scan:
+ *
+ * <pre>
+ * IteratorSetting cfg = new IteratorSetting(priority, "myIter", MyIterator.class);
+ * MyIterator.addOption(cfg, 42);
+ * scanner.addScanIterator(cfg);
+ * </pre>
+ */
+public class IteratorSetting implements Writable {
+ private int priority;
+ private String name;
+ private String iteratorClass;
+ private Map<String,String> properties;
+
+ /**
+ * Get layer at which this iterator applies. See {@link #setPriority(int)} for how the priority is used.
+ *
+ * @return the priority of this Iterator
+ */
+ public int getPriority() {
+ return priority;
+ }
+
+ /**
+ * Set layer at which this iterator applies.
+ *
+ * @param priority
+ * determines the order in which iterators are applied (system iterators are always applied first, then user-configured iterators, lowest priority
+ * first)
+ */
+ public void setPriority(int priority) {
+ ArgumentChecker.strictlyPositive(priority);
+ this.priority = priority;
+ }
+
+ /**
+ * Get the iterator's name.
+ *
+ * @return the name of the iterator
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Set the iterator's name. Must be a simple alphanumeric identifier.
- *
- * @param name
+ */
+ public void setName(String name) {
+ ArgumentChecker.notNull(name);
+ this.name = name;
+ }
+
+ /**
+ * Get the name of the class that implements the iterator.
+ *
+ * @return the iterator's class name
+ */
+ public String getIteratorClass() {
+ return iteratorClass;
+ }
+
+ /**
+ * Set the name of the class that implements the iterator. The class does not have to be present on the client, but it must be available to all tablet
+ * servers.
- *
- * @param iteratorClass
+ */
+ public void setIteratorClass(String iteratorClass) {
+ ArgumentChecker.notNull(iteratorClass);
+ this.iteratorClass = iteratorClass;
+ }
+
+ /**
+ * Constructs an iterator setting configured for the scan scope with no parameters. (Parameters can be added later.)
+ *
+ * @param priority
+ * the priority for the iterator (see {@link #setPriority(int)})
+ * @param name
+ * the distinguishing name for the iterator
+ * @param iteratorClass
+ * the fully qualified class name for the iterator
+ */
+ public IteratorSetting(int priority, String name, String iteratorClass) {
+ this(priority, name, iteratorClass, new HashMap<String,String>());
+ }
+
+ /**
+ * Constructs an iterator setting configured for the specified scopes with the specified parameters.
+ *
+ * @param priority
+ * the priority for the iterator (see {@link #setPriority(int)})
+ * @param name
+ * the distinguishing name for the iterator
+ * @param iteratorClass
+ * the fully qualified class name for the iterator
+ * @param properties
+ * any properties for the iterator
+ */
+ public IteratorSetting(int priority, String name, String iteratorClass, Map<String,String> properties) {
+ setPriority(priority);
+ setName(name);
+ setIteratorClass(iteratorClass);
+ this.properties = new HashMap<String,String>();
+ addOptions(properties);
+ }
+
+ /**
+ * Constructs an iterator setting using the given class's SimpleName for the iterator name. The iterator setting will be configured for the scan scope with no
+ * parameters.
+ *
+ * @param priority
+ * the priority for the iterator (see {@link #setPriority(int)})
+ * @param iteratorClass
+ * the class for the iterator
+ */
+ public IteratorSetting(int priority, Class<? extends SortedKeyValueIterator<Key,Value>> iteratorClass) {
+ this(priority, iteratorClass.getSimpleName(), iteratorClass.getName());
+ }
+
+ /**
+ *
+ * Constructs an iterator setting using the given class's SimpleName for the iterator name and configured for the specified scopes with the specified
+ * parameters.
+ *
+ * @param priority
+ * the priority for the iterator (see {@link #setPriority(int)})
+ * @param iteratorClass
+ * the class for the iterator
+ * @param properties
+ * any properties for the iterator
+ */
+ public IteratorSetting(int priority, Class<? extends SortedKeyValueIterator<Key,Value>> iteratorClass, Map<String,String> properties) {
+ this(priority, iteratorClass.getSimpleName(), iteratorClass.getName(), properties);
+ }
+
+ /**
+ * Constructs an iterator setting configured for the scan scope with no parameters.
+ *
+ * @param priority
+ * the priority for the iterator (see {@link #setPriority(int)})
+ * @param name
+ * the distinguishing name for the iterator
+ * @param iteratorClass
+ * the class for the iterator
+ */
+ public IteratorSetting(int priority, String name, Class<? extends SortedKeyValueIterator<Key,Value>> iteratorClass) {
+ this(priority, name, iteratorClass.getName());
+ }
+
+ /**
+ * @since 1.5.0
+ */
+ public IteratorSetting(DataInput din) throws IOException {
+ this.properties = new HashMap<String,String>();
+ this.readFields(din);
+ }
+
+ /**
+ * Add another option to the iterator.
+ *
+ * @param option
+ * the name of the option
+ * @param value
+ * the value of the option
+ */
+ public void addOption(String option, String value) {
+ ArgumentChecker.notNull(option, value);
+ properties.put(option, value);
+ }
+
+ /**
+ * Remove an option from the iterator.
+ *
+ * @param option
+ * the name of the option
+ * @return the value previously associated with the option, or null if no such option existed
+ */
+ public String removeOption(String option) {
+ ArgumentChecker.notNull(option);
+ return properties.remove(option);
+ }
+
+ /**
+ * Add many options to the iterator.
+ *
+ * @param propertyEntries
+ * a set of entries to add to the options
+ */
+ public void addOptions(Set<Entry<String,String>> propertyEntries) {
+ ArgumentChecker.notNull(propertyEntries);
+ for (Entry<String,String> keyValue : propertyEntries) {
+ addOption(keyValue.getKey(), keyValue.getValue());
+ }
+ }
+
+ /**
+ * Add many options to the iterator.
+ *
+ * @param properties
+ * a map of entries to add to the options
+ */
+ public void addOptions(Map<String,String> properties) {
+ ArgumentChecker.notNull(properties);
+ addOptions(properties.entrySet());
+ }
+
+ /**
+ * Get the configuration parameters for this iterator.
+ *
+ * @return the properties
+ */
+ public Map<String,String> getOptions() {
+ return Collections.unmodifiableMap(properties);
+ }
+
+ /**
+ * Remove all options from the iterator.
+ */
+ public void clearOptions() {
+ properties.clear();
+ }
+
+ /**
+ * @see java.lang.Object#hashCode()
+ */
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((iteratorClass == null) ? 0 : iteratorClass.hashCode());
+ result = prime * result + ((name == null) ? 0 : name.hashCode());
+ result = prime * result + priority;
+ result = prime * result + ((properties == null) ? 0 : properties.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (!(obj instanceof IteratorSetting))
+ return false;
+ IteratorSetting other = (IteratorSetting) obj;
+ if (iteratorClass == null) {
+ if (other.iteratorClass != null)
+ return false;
+ } else if (!iteratorClass.equals(other.iteratorClass))
+ return false;
+ if (name == null) {
+ if (other.name != null)
+ return false;
+ } else if (!name.equals(other.name))
+ return false;
+ if (priority != other.priority)
+ return false;
+ if (properties == null) {
+ if (other.properties != null)
+ return false;
+ } else if (!properties.equals(other.properties))
+ return false;
+ return true;
+ }
+
+ /**
+ * @see java.lang.Object#toString()
+ */
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("name:");
+ sb.append(name);
+ sb.append(", priority:");
+ sb.append(Integer.toString(priority));
+ sb.append(", class:");
+ sb.append(iteratorClass);
+ sb.append(", properties:");
+ sb.append(properties);
+ return sb.toString();
+ }
+
+ /**
+ * A convenience class for passing column family and column qualifiers to iterator configuration methods.
+ */
+ public static class Column extends Pair<Text,Text> {
+
+ public Column(Text columnFamily, Text columnQualifier) {
+ super(columnFamily, columnQualifier);
+ }
+
+ public Column(Text columnFamily) {
+ super(columnFamily, null);
+ }
+
+ public Column(String columnFamily, String columnQualifier) {
+ super(new Text(columnFamily), new Text(columnQualifier));
+ }
+
+ public Column(String columnFamily) {
+ super(new Text(columnFamily), null);
+ }
+
+ public Text getColumnFamily() {
+ return getFirst();
+ }
+
+ public Text getColumnQualifier() {
+ return getSecond();
+ }
+
+ }
+
+ /**
+ * @since 1.5.0
+ */
+ @Override
+ public void readFields(DataInput din) throws IOException {
+ priority = WritableUtils.readVInt(din);
+ name = WritableUtils.readString(din);
+ iteratorClass = WritableUtils.readString(din);
+ properties.clear();
+ int size = WritableUtils.readVInt(din);
+ while (size > 0) {
+ properties.put(WritableUtils.readString(din), WritableUtils.readString(din));
+ size--;
+ }
+ }
+
+ /**
+ * @since 1.5.0
+ */
+ @Override
+ public void write(DataOutput dout) throws IOException {
+ WritableUtils.writeVInt(dout, priority);
+ WritableUtils.writeString(dout, name);
+ WritableUtils.writeString(dout, iteratorClass);
+ WritableUtils.writeVInt(dout, properties.size());
+ for (Entry<String,String> e : properties.entrySet()) {
+ WritableUtils.writeString(dout, e.getKey());
+ WritableUtils.writeString(dout, e.getValue());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/RowIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/RowIterator.java
index 005f697,0000000..f5e9547
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/RowIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/RowIterator.java
@@@ -1,164 -1,0 +1,160 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.NoSuchElementException;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.PeekingIterator;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Group Key/Value pairs into Iterators over rows. Suggested usage:
+ *
+ * <pre>
+ * RowIterator rowIterator = new RowIterator(connector.createScanner(tableName, authorizations));
+ * </pre>
+ */
+public class RowIterator implements Iterator<Iterator<Entry<Key,Value>>> {
+
+ /**
+ * Iterate over entries in a single row.
+ */
+ private static class SingleRowIter implements Iterator<Entry<Key,Value>> {
+ private PeekingIterator<Entry<Key,Value>> source;
+ private Text currentRow = null;
+ private long count = 0;
+ private boolean disabled = false;
+
+ /**
+ * SingleRowIter must be passed a PeekingIterator so that it can peek at the next entry to see if it belongs in the current row or not.
+ */
+ public SingleRowIter(PeekingIterator<Entry<Key,Value>> source) {
+ this.source = source;
+ if (source.hasNext())
+ currentRow = source.peek().getKey().getRow();
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (disabled)
+ throw new IllegalStateException("SingleRowIter no longer valid");
+ return currentRow != null;
+ }
+
+ @Override
+ public Entry<Key,Value> next() {
+ if (disabled)
+ throw new IllegalStateException("SingleRowIter no longer valid");
+ return _next();
+ }
+
+ private Entry<Key,Value> _next() {
+ if (currentRow == null)
+ throw new NoSuchElementException();
+ count++;
+ Entry<Key,Value> kv = source.next();
+ if (!source.hasNext() || !source.peek().getKey().getRow().equals(currentRow)) {
+ currentRow = null;
+ }
+ return kv;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Get a count of entries read from the row (only equals the number of entries in the row when the row has been read fully).
+ */
+ public long getCount() {
+ return count;
+ }
+
+ /**
+ * Consume the rest of the row. Disables the iterator from future use.
+ */
+ public void consume() {
+ disabled = true;
+ while (currentRow != null)
+ _next();
+ }
+ }
+
+ private final PeekingIterator<Entry<Key,Value>> iter;
+ private long count = 0;
+ private SingleRowIter lastIter = null;
+
+ /**
+ * Create an iterator from an (ordered) sequence of KeyValue pairs.
- *
- * @param iterator
+ */
+ public RowIterator(Iterator<Entry<Key,Value>> iterator) {
+ this.iter = new PeekingIterator<Entry<Key,Value>>(iterator);
+ }
+
+ /**
+ * Create an iterator from an Iterable.
- *
- * @param iterable
+ */
+ public RowIterator(Iterable<Entry<Key,Value>> iterable) {
+ this(iterable.iterator());
+ }
+
+ /**
+ * Returns true if there is at least one more row to get.
+ *
+ * If the last row hasn't been fully read, this method will read through the end of the last row so it can determine if the underlying iterator has a next
+ * row. The last row is disabled from future use.
+ */
+ @Override
+ public boolean hasNext() {
+ if (lastIter != null) {
+ lastIter.consume();
+ count += lastIter.getCount();
+ lastIter = null;
+ }
+ return iter.hasNext();
+ }
+
+ /**
+ * Fetch the next row.
+ */
+ @Override
+ public Iterator<Entry<Key,Value>> next() {
+ if (!hasNext())
+ throw new NoSuchElementException();
+ return lastIter = new SingleRowIter(iter);
+ }
+
+ /**
+ * Unsupported.
+ */
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Get a count of the total number of entries in all rows read so far.
+ */
+ public long getKVCount() {
+ return count;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
index 873a3ad,0000000..7c61d57
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
@@@ -1,130 -1,0 +1,130 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.hadoop.io.Text;
+
+/**
+ * This class hosts configuration methods that are shared between different types of scanners.
+ *
+ */
+public interface ScannerBase extends Iterable<Entry<Key,Value>> {
+
+ /**
+ * Add a server-side scan iterator.
+ *
+ * @param cfg
+ * fully specified scan-time iterator, including all options for the iterator. Any changes to the iterator setting after this call are not propagated
+ * to the stored iterator.
+ * @throws IllegalArgumentException
+ * if the setting conflicts with existing iterators
+ */
+ public void addScanIterator(IteratorSetting cfg);
+
+ /**
+ * Remove an iterator from the list of iterators.
+ *
+ * @param iteratorName
+ * nickname used for the iterator
+ */
+ public void removeScanIterator(String iteratorName);
+
+ /**
+ * Update the options for an iterator. Note that this does <b>not</b> change the iterator options during a scan, it just replaces the given option on a
+ * configured iterator before a scan is started.
+ *
+ * @param iteratorName
+ * the name of the iterator to change
+ * @param key
+ * the name of the option
+ * @param value
+ * the new value for the named option
+ */
+ public void updateScanIteratorOption(String iteratorName, String key, String value);
+
+ /**
+ * Adds a column family to the list of columns that will be fetched by this scanner. By default when no columns have been added the scanner fetches all
+ * columns.
+ *
+ * @param col
+ * the column family to be fetched
+ */
+ public void fetchColumnFamily(Text col);
+
+ /**
+ * Adds a column to the list of columns that will be fetched by this scanner. The column is identified by family and qualifier. By default when no columns
+ * have been added the scanner fetches all columns.
+ *
+ * @param colFam
+ * the column family of the column to be fetched
+ * @param colQual
+ * the column qualifier of the column to be fetched
+ */
+ public void fetchColumn(Text colFam, Text colQual);
+
+ /**
+ * Clears the columns to be fetched (useful for resetting the scanner for reuse). Once cleared, the scanner will fetch all columns.
+ */
+ public void clearColumns();
+
+ /**
+ * Clears scan iterators prior to returning a scanner to the pool.
+ */
+ public void clearScanIterators();
+
+ /**
+ * Returns an iterator over an accumulo table. This iterator uses the options that are currently set for its lifetime, so setting options will have no effect
+ * on existing iterators.
+ *
+ * Keys returned by the iterator are not guaranteed to be in sorted order.
+ *
+ * @return an iterator over Key,Value pairs which meet the restrictions set on the scanner
+ */
++ @Override
+ public Iterator<Entry<Key,Value>> iterator();
+
+ /**
+ * This setting determines how long a scanner will automatically retry when a failure occurs. By default a scanner will retry forever.
+ *
+ * Setting to zero or Long.MAX_VALUE and TimeUnit.MILLISECONDS means to retry forever.
+ *
- * @param timeOut
+ * @param timeUnit
+ * determines how timeout is interpreted
+ * @since 1.5.0
+ */
+ public void setTimeout(long timeOut, TimeUnit timeUnit);
+
+ /**
+ * Returns the setting for how long a scanner will automatically retry when a failure occurs.
+ *
+ * @return the timeout configured for this scanner
+ * @since 1.5.0
+ */
+ public long getTimeout(TimeUnit timeUnit);
+
+ /**
+ * Closes any underlying connections on the scanner
+ * @since 1.5.0
+ */
+ public void close();
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
index 5197262,0000000..795ce9f
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
@@@ -1,320 -1,0 +1,318 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.impl.ConnectorImpl;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.OpTimer;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * <p>
+ * An implementation of instance that looks in zookeeper to find information needed to connect to an instance of accumulo.
+ *
+ * <p>
+ * The advantage of using zookeeper to obtain information about accumulo is that zookeeper is highly available, very responsive, and supports caching.
+ *
+ * <p>
+ * Because it is possible for multiple instances of accumulo to share a single set of zookeeper servers, all constructors require an accumulo instance name.
+ *
+ * If you do not know the instance names then run accumulo org.apache.accumulo.server.util.ListInstances on an accumulo server.
+ *
+ */
+
+public class ZooKeeperInstance implements Instance {
+
+ private static final Logger log = Logger.getLogger(ZooKeeperInstance.class);
+
+ private String instanceId = null;
+ private String instanceName = null;
+
+ private final ZooCache zooCache;
+
+ private final String zooKeepers;
+
+ private final int zooKeepersSessionTimeOut;
+
+ /**
+ *
+ * @param instanceName
+ * The name of specific accumulo instance. This is set at initialization time.
+ * @param zooKeepers
+ * A comma separated list of zoo keeper server locations. Each location can contain an optional port, of the format host:port.
+ */
+
+ public ZooKeeperInstance(String instanceName, String zooKeepers) {
+ this(instanceName, zooKeepers, (int) AccumuloConfiguration.getDefaultConfiguration().getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT));
+ }
+
+ /**
+ *
+ * @param instanceName
+ * The name of specific accumulo instance. This is set at initialization time.
+ * @param zooKeepers
+ * A comma separated list of zoo keeper server locations. Each location can contain an optional port, of the format host:port.
+ * @param sessionTimeout
+ * zoo keeper session time out in milliseconds.
+ */
+
+ public ZooKeeperInstance(String instanceName, String zooKeepers, int sessionTimeout) {
+ ArgumentChecker.notNull(instanceName, zooKeepers);
+ this.instanceName = instanceName;
+ this.zooKeepers = zooKeepers;
+ this.zooKeepersSessionTimeOut = sessionTimeout;
+ zooCache = ZooCache.getInstance(zooKeepers, sessionTimeout);
+ getInstanceID();
+ }
+
+ /**
+ *
+ * @param instanceId
+ * The UUID that identifies the accumulo instance you want to connect to.
+ * @param zooKeepers
+ * A comma separated list of zoo keeper server locations. Each location can contain an optional port, of the format host:port.
+ */
+
+ public ZooKeeperInstance(UUID instanceId, String zooKeepers) {
+ this(instanceId, zooKeepers, (int) AccumuloConfiguration.getDefaultConfiguration().getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT));
+ }
+
+ /**
+ *
+ * @param instanceId
+ * The UUID that identifies the accumulo instance you want to connect to.
+ * @param zooKeepers
+ * A comma separated list of zoo keeper server locations. Each location can contain an optional port, of the format host:port.
+ * @param sessionTimeout
+ * zoo keeper session time out in milliseconds.
+ */
+
+ public ZooKeeperInstance(UUID instanceId, String zooKeepers, int sessionTimeout) {
+ ArgumentChecker.notNull(instanceId, zooKeepers);
+ this.instanceId = instanceId.toString();
+ this.zooKeepers = zooKeepers;
+ this.zooKeepersSessionTimeOut = sessionTimeout;
+ zooCache = ZooCache.getInstance(zooKeepers, sessionTimeout);
+ }
+
+ @Override
+ public String getInstanceID() {
+ if (instanceId == null) {
+ // want the instance id to be stable for the life of this instance object,
+ // so only get it once
+ String instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + instanceName;
+ byte[] iidb = zooCache.get(instanceNamePath);
+ if (iidb == null) {
+ throw new RuntimeException("Instance name " + instanceName
+ + " does not exist in zookeeper. Run \"accumulo org.apache.accumulo.server.util.ListInstances\" to see a list.");
+ }
+ instanceId = new String(iidb, Constants.UTF8);
+ }
+
+ if (zooCache.get(Constants.ZROOT + "/" + instanceId) == null) {
+ if (instanceName == null)
+ throw new RuntimeException("Instance id " + instanceId + " does not exist in zookeeper");
+ throw new RuntimeException("Instance id " + instanceId + " pointed to by the name " + instanceName + " does not exist in zookeeper");
+ }
+
+ return instanceId;
+ }
+
+ @Override
+ public List<String> getMasterLocations() {
+ String masterLocPath = ZooUtil.getRoot(this) + Constants.ZMASTER_LOCK;
+
+ OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Looking up master location in zoocache.");
+ byte[] loc = ZooUtil.getLockData(zooCache, masterLocPath);
+ opTimer.stop("Found master at " + (loc == null ? null : new String(loc, Constants.UTF8)) + " in %DURATION%");
+
+ if (loc == null) {
+ return Collections.emptyList();
+ }
+
+ return Collections.singletonList(new String(loc, Constants.UTF8));
+ }
+
+ @Override
+ public String getRootTabletLocation() {
+ String zRootLocPath = ZooUtil.getRoot(this) + Constants.ZROOT_TABLET_LOCATION;
+
+ OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Looking up root tablet location in zookeeper.");
+ byte[] loc = zooCache.get(zRootLocPath);
+ opTimer.stop("Found root tablet at " + (loc == null ? null : new String(loc, Constants.UTF8)) + " in %DURATION%");
+
+ if (loc == null) {
+ return null;
+ }
+
+ return new String(loc, Constants.UTF8).split("\\|")[0];
+ }
+
+ @Override
+ public String getInstanceName() {
+ if (instanceName == null)
+ instanceName = lookupInstanceName(zooCache, UUID.fromString(getInstanceID()));
+
+ return instanceName;
+ }
+
+ @Override
+ public String getZooKeepers() {
+ return zooKeepers;
+ }
+
+ @Override
+ public int getZooKeepersSessionTimeOut() {
+ return zooKeepersSessionTimeOut;
+ }
+
+ @Override
+ @Deprecated
+ public Connector getConnector(String user, CharSequence pass) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(user, TextUtil.getBytes(new Text(pass.toString())));
+ }
+
+ @Override
+ @Deprecated
+ public Connector getConnector(String user, ByteBuffer pass) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(user, ByteBufferUtil.toBytes(pass));
+ }
+
+ @Override
+ public Connector getConnector(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(CredentialHelper.create(principal, token, getInstanceID()));
+ }
+
+ @SuppressWarnings("deprecation")
+ private Connector getConnector(TCredentials credential) throws AccumuloException, AccumuloSecurityException {
+ return new ConnectorImpl(this, credential);
+ }
+
+ @Override
+ @Deprecated
+ public Connector getConnector(String principal, byte[] pass) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(principal, new PasswordToken(pass));
+ }
+
+ private AccumuloConfiguration conf = null;
+
+ @Override
+ public AccumuloConfiguration getConfiguration() {
+ if (conf == null)
+ conf = AccumuloConfiguration.getDefaultConfiguration();
+ return conf;
+ }
+
+ @Override
+ public void setConfiguration(AccumuloConfiguration conf) {
+ this.conf = conf;
+ }
+
+ /**
+ * @deprecated Use {@link #lookupInstanceName(org.apache.accumulo.fate.zookeeper.ZooCache, UUID)} instead
+ */
+ @Deprecated
+ public static String lookupInstanceName(org.apache.accumulo.core.zookeeper.ZooCache zooCache, UUID instanceId) {
+ return lookupInstanceName((ZooCache) zooCache, instanceId);
+ }
+
+ /**
+ * Given a zooCache and instanceId, look up the instance name.
+ *
- * @param zooCache
- * @param instanceId
+ * @return the instance name
+ */
+ public static String lookupInstanceName(ZooCache zooCache, UUID instanceId) {
+ ArgumentChecker.notNull(zooCache, instanceId);
+ for (String name : zooCache.getChildren(Constants.ZROOT + Constants.ZINSTANCES)) {
+ String instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + name;
+ byte[] bytes = zooCache.get(instanceNamePath);
+ UUID iid = UUID.fromString(new String(bytes, Constants.UTF8));
+ if (iid.equals(instanceId)) {
+ return name;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * To be moved to server code. Only lives here to support certain client side utilities to minimize command-line options.
+ */
+ @Deprecated
+ public static String getInstanceIDFromHdfs(Path instanceDirectory) {
+ try {
+ FileSystem fs = FileUtil.getFileSystem(CachedConfiguration.getInstance(), AccumuloConfiguration.getSiteConfiguration());
+ FileStatus[] files = null;
+ try {
+ files = fs.listStatus(instanceDirectory);
+ } catch (FileNotFoundException ex) {
+ // ignored
+ }
+ log.debug("Trying to read instance id from " + instanceDirectory);
+ if (files == null || files.length == 0) {
+ log.error("unable obtain instance id at " + instanceDirectory);
+ throw new RuntimeException("Accumulo not initialized, there is no instance id at " + instanceDirectory);
+ } else if (files.length != 1) {
+ log.error("multiple potential instances in " + instanceDirectory);
+ throw new RuntimeException("Accumulo found multiple possible instance ids in " + instanceDirectory);
+ } else {
+ String result = files[0].getPath().getName();
+ return result;
+ }
+ } catch (IOException e) {
+ log.error("Problem reading instance id out of hdfs at " + instanceDirectory, e);
+ throw new RuntimeException("Can't tell if Accumulo is initialized; can't read instance id at " + instanceDirectory, e);
+ } catch (IllegalArgumentException exception) {
+ /* HDFS throws this when there's a UnknownHostException due to DNS troubles. */
+ if (exception.getCause() instanceof UnknownHostException) {
+ log.error("Problem reading instance id out of hdfs at " + instanceDirectory, exception);
+ }
+ throw exception;
+ }
+ }
+
+ @Deprecated
+ @Override
+ public Connector getConnector(org.apache.accumulo.core.security.thrift.AuthInfo auth) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(auth.user, auth.password);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/admin/ActiveCompaction.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/admin/ActiveCompaction.java
index f2876b2,0000000..9c39ea6
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/ActiveCompaction.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/ActiveCompaction.java
@@@ -1,185 -1,0 +1,184 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.admin;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+
+
+/**
+ *
+ * @since 1.5.0
+ */
+public class ActiveCompaction {
+
+ private org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction tac;
+ private Instance instance;
+
+ ActiveCompaction(Instance instance, org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction tac) {
+ this.tac = tac;
+ this.instance = instance;
+ }
+
+ public static enum CompactionType {
+ /**
+ * compaction to flush a tablets memory
+ */
+ MINOR,
+ /**
+ * compaction to flush a tablets memory and merge it with the tablets smallest file. This type compaction is done when a tablet has too many files
+ */
+ MERGE,
+ /**
+ * compaction that merges a subset of a tablets files into one file
+ */
+ MAJOR,
+ /**
+ * compaction that merges all of a tablets files into one file
+ */
+ FULL
+ };
+
+ public static enum CompactionReason {
+ /**
+ * compaction initiated by user
+ */
+ USER,
+ /**
+ * Compaction initiated by system
+ */
+ SYSTEM,
+ /**
+ * Compaction initiated by merge operation
+ */
+ CHOP,
+ /**
+ * idle compaction
+ */
+ IDLE,
+ /**
+ * Compaction initiated to close a unload a tablet
+ */
+ CLOSE
+ };
+
+ /**
+ *
+ * @return name of the table the compaction is running against
- * @throws TableNotFoundException
+ */
+
+ public String getTable() throws TableNotFoundException {
+ return Tables.getTableName(instance, getExtent().getTableId().toString());
+ }
+
+ /**
+ * @return tablet thats is compacting
+ */
+
+ public KeyExtent getExtent() {
+ return new KeyExtent(tac.getExtent());
+ }
+
+ /**
+ * @return how long the compaction has been running in milliseconds
+ */
+
+ public long getAge() {
+ return tac.getAge();
+ }
+
+ /**
+ * @return the files the compaction is reading from
+ */
+
+ public List<String> getInputFiles() {
+ return tac.getInputFiles();
+ }
+
+ /**
+ * @return file compactions is writing too
+ */
+
+ public String getOutputFile() {
+ return tac.getOutputFile();
+ }
+
+ /**
+ * @return the type of compaction
+ */
+ public CompactionType getType() {
+ return CompactionType.valueOf(tac.getType().name());
+ }
+
+ /**
+ * @return the reason the compaction was started
+ */
+
+ public CompactionReason getReason() {
+ return CompactionReason.valueOf(tac.getReason().name());
+ }
+
+ /**
+ * @return the locality group that is compacting
+ */
+
+ public String getLocalityGroup() {
+ return tac.getLocalityGroup();
+ }
+
+ /**
+ * @return the number of key/values read by the compaction
+ */
+
+ public long getEntriesRead() {
+ return tac.getEntriesRead();
+ }
+
+ /**
+ * @return the number of key/values written by the compaction
+ */
+
+ public long getEntriesWritten() {
+ return tac.getEntriesWritten();
+ }
+
+ /**
+ * @return the per compaction iterators configured
+ */
+
+ public List<IteratorSetting> getIterators() {
+ ArrayList<IteratorSetting> ret = new ArrayList<IteratorSetting>();
+
+ for (IterInfo ii : tac.getSsiList()) {
+ IteratorSetting settings = new IteratorSetting(ii.getPriority(), ii.getIterName(), ii.getClassName());
+ Map<String,String> options = tac.getSsio().get(ii.getIterName());
+ settings.addOptions(options);
+
+ ret.add(settings);
+ }
+
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
index fce0716,0000000..29ff2a6
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
@@@ -1,131 -1,0 +1,119 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.admin;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+
+/**
+ *
+ */
+public interface InstanceOperations {
+
+ /**
+ * Sets an system property in zookeeper. Tablet servers will pull this setting and override the equivalent setting in accumulo-site.xml. Changes can be seen
+ * using {@link #getSystemConfiguration()}
+ *
+ * @param property
+ * the name of a per-table property
+ * @param value
+ * the value to set a per-table property to
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ public void setProperty(final String property, final String value) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Removes a system property from zookeeper. Changes can be seen using {@link #getSystemConfiguration()}
+ *
+ * @param property
+ * the name of a per-table property
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ public void removeProperty(final String property) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ *
+ * @return A map of system properties set in zookeeper. If a property is not set in zookeeper, then it will return the value set in accumulo-site.xml on some
+ * server. If nothing is set in an accumulo-site.xml file it will return the default value for each property.
- * @throws AccumuloException
- * @throws AccumuloSecurityException
+ */
+
+ public Map<String,String> getSystemConfiguration() throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ *
+ * @return A map of system properties set in accumulo-site.xml on some server. If nothing is set in an accumulo-site.xml file it will return the default value
+ * for each property.
- * @throws AccumuloException
- * @throws AccumuloSecurityException
+ */
+
+ public Map<String,String> getSiteConfiguration() throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * List the currently active tablet servers participating in the accumulo instance
+ *
+ * @return A list of currently active tablet servers.
+ */
+
+ public List<String> getTabletServers();
+
+ /**
+ * List the active scans on tablet server.
+ *
+ * @param tserver
+ * The tablet server address should be of the form <ip address>:<port>
+ * @return A list of active scans on tablet server.
- * @throws AccumuloException
- * @throws AccumuloSecurityException
+ */
+
+ public List<ActiveScan> getActiveScans(String tserver) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * List the active compaction running on a tablet server
+ *
+ * @param tserver
+ * The tablet server address should be of the form <ip address>:<port>
+ * @return the list of active compactions
- * @throws AccumuloException
- * @throws AccumuloSecurityException
+ * @since 1.5.0
+ */
+
+ public List<ActiveCompaction> getActiveCompactions(String tserver) throws AccumuloException, AccumuloSecurityException;
+
+ /**
+ * Throws an exception if a tablet server can not be contacted.
+ *
+ * @param tserver
+ * The tablet server address should be of the form <ip address>:<port>
- * @throws AccumuloException
+ * @since 1.5.0
+ */
+ public void ping(String tserver) throws AccumuloException;
+
+ /**
+ * Test to see if the instance can load the given class as the given type. This check does not consider per table classpaths, see
+ * {@link TableOperations#testClassLoad(String, String, String)}
+ *
- * @param className
- * @param asTypeName
+ * @return true if the instance can load the given class as the given type, false otherwise
- * @throws AccumuloException
+ */
+ public boolean testClassLoad(final String className, final String asTypeName) throws AccumuloException, AccumuloSecurityException;
+
+}
[26/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFileDumper.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFileDumper.java
index 9b9cd51,0000000..4b3e41c
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFileDumper.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFileDumper.java
@@@ -1,259 -1,0 +1,258 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.accumulo.core.file.rfile.bcfile;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.file.rfile.bcfile.BCFile.BlockRegion;
+import org.apache.accumulo.core.file.rfile.bcfile.BCFile.MetaIndexEntry;
+import org.apache.accumulo.core.file.rfile.bcfile.TFile.TFileIndexEntry;
+import org.apache.accumulo.core.file.rfile.bcfile.Utils.Version;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+
+/**
+ * Dumping the information of a TFile.
+ */
+class TFileDumper {
+ static final Log LOG = LogFactory.getLog(TFileDumper.class);
+
+ private TFileDumper() {
+ // namespace object not constructable.
+ }
+
+ private enum Align {
+ LEFT, CENTER, RIGHT, ZERO_PADDED;
+ static String format(String s, int width, Align align) {
+ if (s.length() >= width)
+ return s;
+ int room = width - s.length();
+ Align alignAdjusted = align;
+ if (room == 1) {
+ alignAdjusted = LEFT;
+ }
+ if (alignAdjusted == LEFT) {
+ return s + String.format("%" + room + "s", "");
+ }
+ if (alignAdjusted == RIGHT) {
+ return String.format("%" + room + "s", "") + s;
+ }
+ if (alignAdjusted == CENTER) {
+ int half = room / 2;
+ return String.format("%" + half + "s", "") + s + String.format("%" + (room - half) + "s", "");
+ }
+ throw new IllegalArgumentException("Unsupported alignment");
+ }
+
+ static String format(long l, int width, Align align) {
+ if (align == ZERO_PADDED) {
+ return String.format("%0" + width + "d", l);
+ }
+ return format(Long.toString(l), width, align);
+ }
+
+ static int calculateWidth(String caption, long max) {
+ return Math.max(caption.length(), Long.toString(max).length());
+ }
+ }
+
+ /**
+ * Dump information about TFile.
+ *
+ * @param file
+ * Path string of the TFile
+ * @param out
+ * PrintStream to output the information.
+ * @param conf
+ * The configuration object.
- * @throws IOException
+ */
+ static public void dumpInfo(String file, PrintStream out, Configuration conf) throws IOException {
+ final int maxKeySampleLen = 16;
+ Path path = new Path(file);
+ FileSystem fs = path.getFileSystem(conf);
+ long length = fs.getFileStatus(path).getLen();
+ FSDataInputStream fsdis = fs.open(path);
+ TFile.Reader reader = new TFile.Reader(fsdis, length, conf);
+ try {
+ LinkedHashMap<String,String> properties = new LinkedHashMap<String,String>();
+ int blockCnt = reader.readerBCF.getBlockCount();
+ int metaBlkCnt = reader.readerBCF.metaIndex.index.size();
+ properties.put("BCFile Version", reader.readerBCF.version.toString());
+ properties.put("TFile Version", reader.tfileMeta.version.toString());
+ properties.put("File Length", Long.toString(length));
+ properties.put("Data Compression", reader.readerBCF.getDefaultCompressionName());
+ properties.put("Record Count", Long.toString(reader.getEntryCount()));
+ properties.put("Sorted", Boolean.toString(reader.isSorted()));
+ if (reader.isSorted()) {
+ properties.put("Comparator", reader.getComparatorName());
+ }
+ properties.put("Data Block Count", Integer.toString(blockCnt));
+ long dataSize = 0, dataSizeUncompressed = 0;
+ if (blockCnt > 0) {
+ for (int i = 0; i < blockCnt; ++i) {
+ BlockRegion region = reader.readerBCF.dataIndex.getBlockRegionList().get(i);
+ dataSize += region.getCompressedSize();
+ dataSizeUncompressed += region.getRawSize();
+ }
+ properties.put("Data Block Bytes", Long.toString(dataSize));
+ if (reader.readerBCF.getDefaultCompressionName() != "none") {
+ properties.put("Data Block Uncompressed Bytes", Long.toString(dataSizeUncompressed));
+ properties.put("Data Block Compression Ratio", String.format("1:%.1f", (double) dataSizeUncompressed / dataSize));
+ }
+ }
+
+ properties.put("Meta Block Count", Integer.toString(metaBlkCnt));
+ long metaSize = 0, metaSizeUncompressed = 0;
+ if (metaBlkCnt > 0) {
+ Collection<MetaIndexEntry> metaBlks = reader.readerBCF.metaIndex.index.values();
+ boolean calculateCompression = false;
+ for (Iterator<MetaIndexEntry> it = metaBlks.iterator(); it.hasNext();) {
+ MetaIndexEntry e = it.next();
+ metaSize += e.getRegion().getCompressedSize();
+ metaSizeUncompressed += e.getRegion().getRawSize();
+ if (e.getCompressionAlgorithm() != Compression.Algorithm.NONE) {
+ calculateCompression = true;
+ }
+ }
+ properties.put("Meta Block Bytes", Long.toString(metaSize));
+ if (calculateCompression) {
+ properties.put("Meta Block Uncompressed Bytes", Long.toString(metaSizeUncompressed));
+ properties.put("Meta Block Compression Ratio", String.format("1:%.1f", (double) metaSizeUncompressed / metaSize));
+ }
+ }
+ properties.put("Meta-Data Size Ratio", String.format("1:%.1f", (double) dataSize / metaSize));
+ long leftOverBytes = length - dataSize - metaSize;
+ long miscSize = BCFile.Magic.size() * 2 + Long.SIZE / Byte.SIZE + Version.size();
+ long metaIndexSize = leftOverBytes - miscSize;
+ properties.put("Meta Block Index Bytes", Long.toString(metaIndexSize));
+ properties.put("Headers Etc Bytes", Long.toString(miscSize));
+ // Now output the properties table.
+ int maxKeyLength = 0;
+ Set<Map.Entry<String,String>> entrySet = properties.entrySet();
+ for (Iterator<Map.Entry<String,String>> it = entrySet.iterator(); it.hasNext();) {
+ Map.Entry<String,String> e = it.next();
+ if (e.getKey().length() > maxKeyLength) {
+ maxKeyLength = e.getKey().length();
+ }
+ }
+ for (Iterator<Map.Entry<String,String>> it = entrySet.iterator(); it.hasNext();) {
+ Map.Entry<String,String> e = it.next();
+ out.printf("%s : %s%n", Align.format(e.getKey(), maxKeyLength, Align.LEFT), e.getValue());
+ }
+ out.println();
+ reader.checkTFileDataIndex();
+ if (blockCnt > 0) {
+ String blkID = "Data-Block";
+ int blkIDWidth = Align.calculateWidth(blkID, blockCnt);
+ int blkIDWidth2 = Align.calculateWidth("", blockCnt);
+ String offset = "Offset";
+ int offsetWidth = Align.calculateWidth(offset, length);
+ String blkLen = "Length";
+ int blkLenWidth = Align.calculateWidth(blkLen, dataSize / blockCnt * 10);
+ String rawSize = "Raw-Size";
+ int rawSizeWidth = Align.calculateWidth(rawSize, dataSizeUncompressed / blockCnt * 10);
+ String records = "Records";
+ int recordsWidth = Align.calculateWidth(records, reader.getEntryCount() / blockCnt * 10);
+ String endKey = "End-Key";
+ int endKeyWidth = Math.max(endKey.length(), maxKeySampleLen * 2 + 5);
+
+ out.printf("%s %s %s %s %s %s%n", Align.format(blkID, blkIDWidth, Align.CENTER), Align.format(offset, offsetWidth, Align.CENTER),
+ Align.format(blkLen, blkLenWidth, Align.CENTER), Align.format(rawSize, rawSizeWidth, Align.CENTER),
+ Align.format(records, recordsWidth, Align.CENTER), Align.format(endKey, endKeyWidth, Align.LEFT));
+
+ for (int i = 0; i < blockCnt; ++i) {
+ BlockRegion region = reader.readerBCF.dataIndex.getBlockRegionList().get(i);
+ TFileIndexEntry indexEntry = reader.tfileIndex.getEntry(i);
+ out.printf("%s %s %s %s %s ", Align.format(Align.format(i, blkIDWidth2, Align.ZERO_PADDED), blkIDWidth, Align.LEFT),
+ Align.format(region.getOffset(), offsetWidth, Align.LEFT), Align.format(region.getCompressedSize(), blkLenWidth, Align.LEFT),
+ Align.format(region.getRawSize(), rawSizeWidth, Align.LEFT), Align.format(indexEntry.kvEntries, recordsWidth, Align.LEFT));
+ byte[] key = indexEntry.key;
+ boolean asAscii = true;
+ int sampleLen = Math.min(maxKeySampleLen, key.length);
+ for (int j = 0; j < sampleLen; ++j) {
+ byte b = key[j];
+ if ((b < 32 && b != 9) || (b == 127)) {
+ asAscii = false;
+ }
+ }
+ if (!asAscii) {
+ out.print("0X");
+ for (int j = 0; j < sampleLen; ++j) {
+ byte b = key[i];
+ out.printf("%X", b);
+ }
+ } else {
+ out.print(new String(key, 0, sampleLen, Constants.UTF8));
+ }
+ if (sampleLen < key.length) {
+ out.print("...");
+ }
+ out.println();
+ }
+ }
+
+ out.println();
+ if (metaBlkCnt > 0) {
+ String name = "Meta-Block";
+ int maxNameLen = 0;
+ Set<Map.Entry<String,MetaIndexEntry>> metaBlkEntrySet = reader.readerBCF.metaIndex.index.entrySet();
+ for (Iterator<Map.Entry<String,MetaIndexEntry>> it = metaBlkEntrySet.iterator(); it.hasNext();) {
+ Map.Entry<String,MetaIndexEntry> e = it.next();
+ if (e.getKey().length() > maxNameLen) {
+ maxNameLen = e.getKey().length();
+ }
+ }
+ int nameWidth = Math.max(name.length(), maxNameLen);
+ String offset = "Offset";
+ int offsetWidth = Align.calculateWidth(offset, length);
+ String blkLen = "Length";
+ int blkLenWidth = Align.calculateWidth(blkLen, metaSize / metaBlkCnt * 10);
+ String rawSize = "Raw-Size";
+ int rawSizeWidth = Align.calculateWidth(rawSize, metaSizeUncompressed / metaBlkCnt * 10);
+ String compression = "Compression";
+ int compressionWidth = compression.length();
+ out.printf("%s %s %s %s %s%n", Align.format(name, nameWidth, Align.CENTER), Align.format(offset, offsetWidth, Align.CENTER),
+ Align.format(blkLen, blkLenWidth, Align.CENTER), Align.format(rawSize, rawSizeWidth, Align.CENTER),
+ Align.format(compression, compressionWidth, Align.LEFT));
+
+ for (Iterator<Map.Entry<String,MetaIndexEntry>> it = metaBlkEntrySet.iterator(); it.hasNext();) {
+ Map.Entry<String,MetaIndexEntry> e = it.next();
+ String blkName = e.getValue().getMetaName();
+ BlockRegion region = e.getValue().getRegion();
+ String blkCompression = e.getValue().getCompressionAlgorithm().getName();
+ out.printf("%s %s %s %s %s%n", Align.format(blkName, nameWidth, Align.LEFT), Align.format(region.getOffset(), offsetWidth, Align.LEFT),
+ Align.format(region.getCompressedSize(), blkLenWidth, Align.LEFT), Align.format(region.getRawSize(), rawSizeWidth, Align.LEFT),
+ Align.format(blkCompression, compressionWidth, Align.LEFT));
+ }
+ }
+ } finally {
+ IOUtils.cleanup(LOG, reader, fsdis);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Utils.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Utils.java
index 45a59f6,0000000..9131d30
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Utils.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Utils.java
@@@ -1,485 -1,0 +1,474 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.accumulo.core.file.rfile.bcfile;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.hadoop.io.Text;
+
+/**
+ * Supporting Utility classes used by TFile, and shared by users of TFile.
+ */
+public final class Utils {
+
+ /**
+ * Prevent the instantiation of Utils.
+ */
+ private Utils() {
+ // nothing
+ }
+
+ /**
+ * Encoding an integer into a variable-length encoding format. Synonymous to <code>Utils#writeVLong(out, n)</code>.
+ *
+ * @param out
+ * output stream
+ * @param n
+ * The integer to be encoded
- * @throws IOException
+ * @see Utils#writeVLong(DataOutput, long)
+ */
+ public static void writeVInt(DataOutput out, int n) throws IOException {
+ writeVLong(out, n);
+ }
+
+ /**
+ * Encoding a Long integer into a variable-length encoding format.
+ * <ul>
+ * <li>if n in [-32, 127): encode in one byte with the actual value. Otherwise,
+ * <li>if n in [-20*2^8, 20*2^8): encode in two bytes: byte[0] = n/256 - 52; byte[1]=n&0xff. Otherwise,
+ * <li>if n IN [-16*2^16, 16*2^16): encode in three bytes: byte[0]=n/2^16 - 88; byte[1]=(n>>8)&0xff; byte[2]=n&0xff. Otherwise,
+ * <li>if n in [-8*2^24, 8*2^24): encode in four bytes: byte[0]=n/2^24 - 112; byte[1] = (n>>16)&0xff; byte[2] = (n>>8)&0xff; byte[3]=n&0xff. Otherwise:
+ * <li>if n in [-2^31, 2^31): encode in five bytes: byte[0]=-125; byte[1] = (n>>24)&0xff; byte[2]=(n>>16)&0xff; byte[3]=(n>>8)&0xff; byte[4]=n&0xff;
+ * <li>if n in [-2^39, 2^39): encode in six bytes: byte[0]=-124; byte[1] = (n>>32)&0xff; byte[2]=(n>>24)&0xff; byte[3]=(n>>16)&0xff; byte[4]=(n>>8)&0xff;
+ * byte[5]=n&0xff
+ * <li>if n in [-2^47, 2^47): encode in seven bytes: byte[0]=-123; byte[1] = (n>>40)&0xff; byte[2]=(n>>32)&0xff; byte[3]=(n>>24)&0xff; byte[4]=(n>>16)&0xff;
+ * byte[5]=(n>>8)&0xff; byte[6]=n&0xff;
+ * <li>if n in [-2^55, 2^55): encode in eight bytes: byte[0]=-122; byte[1] = (n>>48)&0xff; byte[2] = (n>>40)&0xff; byte[3]=(n>>32)&0xff; byte[4]=(n>>24)&0xff;
+ * byte[5]=(n>>16)&0xff; byte[6]=(n>>8)&0xff; byte[7]=n&0xff;
+ * <li>if n in [-2^63, 2^63): encode in nine bytes: byte[0]=-121; byte[1] = (n>>54)&0xff; byte[2] = (n>>48)&0xff; byte[3] = (n>>40)&0xff;
+ * byte[4]=(n>>32)&0xff; byte[5]=(n>>24)&0xff; byte[6]=(n>>16)&0xff; byte[7]=(n>>8)&0xff; byte[8]=n&0xff;
+ * </ul>
+ *
+ * @param out
+ * output stream
+ * @param n
+ * the integer number
- * @throws IOException
+ */
+ @SuppressWarnings("fallthrough")
+ public static void writeVLong(DataOutput out, long n) throws IOException {
+ if ((n < 128) && (n >= -32)) {
+ out.writeByte((int) n);
+ return;
+ }
+
+ long un = (n < 0) ? ~n : n;
+ // how many bytes do we need to represent the number with sign bit?
+ int len = (Long.SIZE - Long.numberOfLeadingZeros(un)) / 8 + 1;
+ int firstByte = (int) (n >> ((len - 1) * 8));
+ switch (len) {
+ case 1:
+ // fall it through to firstByte==-1, len=2.
+ firstByte >>= 8;
+ case 2:
+ if ((firstByte < 20) && (firstByte >= -20)) {
+ out.writeByte(firstByte - 52);
+ out.writeByte((int) n);
+ return;
+ }
+ // fall it through to firstByte==0/-1, len=3.
+ firstByte >>= 8;
+ case 3:
+ if ((firstByte < 16) && (firstByte >= -16)) {
+ out.writeByte(firstByte - 88);
+ out.writeShort((int) n);
+ return;
+ }
+ // fall it through to firstByte==0/-1, len=4.
+ firstByte >>= 8;
+ case 4:
+ if ((firstByte < 8) && (firstByte >= -8)) {
+ out.writeByte(firstByte - 112);
+ out.writeShort(((int) n) >>> 8);
+ out.writeByte((int) n);
+ return;
+ }
+ out.writeByte(len - 129);
+ out.writeInt((int) n);
+ return;
+ case 5:
+ out.writeByte(len - 129);
+ out.writeInt((int) (n >>> 8));
+ out.writeByte((int) n);
+ return;
+ case 6:
+ out.writeByte(len - 129);
+ out.writeInt((int) (n >>> 16));
+ out.writeShort((int) n);
+ return;
+ case 7:
+ out.writeByte(len - 129);
+ out.writeInt((int) (n >>> 24));
+ out.writeShort((int) (n >>> 8));
+ out.writeByte((int) n);
+ return;
+ case 8:
+ out.writeByte(len - 129);
+ out.writeLong(n);
+ return;
+ default:
+ throw new RuntimeException("Internel error");
+ }
+ }
+
+ /**
+ * Decoding the variable-length integer. Synonymous to <code>(int)Utils#readVLong(in)</code>.
+ *
+ * @param in
+ * input stream
+ * @return the decoded integer
- * @throws IOException
+ *
+ * @see Utils#readVLong(DataInput)
+ */
+ public static int readVInt(DataInput in) throws IOException {
+ long ret = readVLong(in);
+ if ((ret > Integer.MAX_VALUE) || (ret < Integer.MIN_VALUE)) {
+ throw new RuntimeException("Number too large to be represented as Integer");
+ }
+ return (int) ret;
+ }
+
+ /**
+ * Decoding the variable-length integer. Suppose the value of the first byte is FB, and the following bytes are NB[*].
+ * <ul>
+ * <li>if (FB >= -32), return (long)FB;
+ * <li>if (FB in [-72, -33]), return (FB+52)<<8 + NB[0]&0xff;
+ * <li>if (FB in [-104, -73]), return (FB+88)<<16 + (NB[0]&0xff)<<8 + NB[1]&0xff;
+ * <li>if (FB in [-120, -105]), return (FB+112)<<24 + (NB[0]&0xff)<<16 + (NB[1]&0xff)<<8 + NB[2]&0xff;
+ * <li>if (FB in [-128, -121]), return interpret NB[FB+129] as a signed big-endian integer.
+ *
+ * @param in
+ * input stream
+ * @return the decoded long integer.
- * @throws IOException
+ */
+
+ public static long readVLong(DataInput in) throws IOException {
+ int firstByte = in.readByte();
+ if (firstByte >= -32) {
+ return firstByte;
+ }
+
+ switch ((firstByte + 128) / 8) {
+ case 11:
+ case 10:
+ case 9:
+ case 8:
+ case 7:
+ return ((firstByte + 52) << 8) | in.readUnsignedByte();
+ case 6:
+ case 5:
+ case 4:
+ case 3:
+ return ((firstByte + 88) << 16) | in.readUnsignedShort();
+ case 2:
+ case 1:
+ return ((firstByte + 112) << 24) | (in.readUnsignedShort() << 8) | in.readUnsignedByte();
+ case 0:
+ int len = firstByte + 129;
+ switch (len) {
+ case 4:
+ return in.readInt();
+ case 5:
+ return ((long) in.readInt()) << 8 | in.readUnsignedByte();
+ case 6:
+ return ((long) in.readInt()) << 16 | in.readUnsignedShort();
+ case 7:
+ return ((long) in.readInt()) << 24 | (in.readUnsignedShort() << 8) | in.readUnsignedByte();
+ case 8:
+ return in.readLong();
+ default:
+ throw new IOException("Corrupted VLong encoding");
+ }
+ default:
+ throw new RuntimeException("Internal error");
+ }
+ }
+
+ /**
+ * Write a String as a VInt n, followed by n Bytes as in Text format.
- *
- * @param out
- * @param s
- * @throws IOException
+ */
+ public static void writeString(DataOutput out, String s) throws IOException {
+ if (s != null) {
+ Text text = new Text(s);
+ byte[] buffer = text.getBytes();
+ int len = text.getLength();
+ writeVInt(out, len);
+ out.write(buffer, 0, len);
+ } else {
+ writeVInt(out, -1);
+ }
+ }
+
+ /**
+ * Read a String as a VInt n, followed by n Bytes in Text format.
+ *
+ * @param in
+ * The input stream.
+ * @return The string
- * @throws IOException
+ */
+ public static String readString(DataInput in) throws IOException {
+ int length = readVInt(in);
+ if (length == -1)
+ return null;
+ byte[] buffer = new byte[length];
+ in.readFully(buffer);
+ return Text.decode(buffer);
+ }
+
+ /**
+ * A generic Version class. We suggest applications built on top of TFile use this class to maintain version information in their meta blocks.
+ *
+ * A version number consists of a major version and a minor version. The suggested usage of major and minor version number is to increment major version
+ * number when the new storage format is not backward compatible, and increment the minor version otherwise.
+ */
+ public static final class Version implements Comparable<Version> {
+ private final short major;
+ private final short minor;
+
+ /**
+ * Construct the Version object by reading from the input stream.
+ *
+ * @param in
+ * input stream
- * @throws IOException
+ */
+ public Version(DataInput in) throws IOException {
+ major = in.readShort();
+ minor = in.readShort();
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param major
+ * major version.
+ * @param minor
+ * minor version.
+ */
+ public Version(short major, short minor) {
+ this.major = major;
+ this.minor = minor;
+ }
+
+ /**
+ * Write the object to a DataOutput. The serialized format of the Version is major version followed by minor version, both as big-endian short integers.
+ *
+ * @param out
+ * The DataOutput object.
- * @throws IOException
+ */
+ public void write(DataOutput out) throws IOException {
+ out.writeShort(major);
+ out.writeShort(minor);
+ }
+
+ /**
+ * Get the major version.
+ *
+ * @return Major version.
+ */
+ public int getMajor() {
+ return major;
+ }
+
+ /**
+ * Get the minor version.
+ *
+ * @return The minor version.
+ */
+ public int getMinor() {
+ return minor;
+ }
+
+ /**
+ * Get the size of the serialized Version object.
+ *
+ * @return serialized size of the version object.
+ */
+ public static int size() {
+ return (Short.SIZE + Short.SIZE) / Byte.SIZE;
+ }
+
+ /**
+ * Return a string representation of the version.
+ */
+ @Override
+ public String toString() {
+ return new StringBuilder("v").append(major).append(".").append(minor).toString();
+ }
+
+ /**
+ * Test compatibility.
+ *
+ * @param other
+ * The Version object to test compatibility with.
+ * @return true if both versions have the same major version number; false otherwise.
+ */
+ public boolean compatibleWith(Version other) {
+ return major == other.major;
+ }
+
+ /**
+ * Compare this version with another version.
+ */
+ @Override
+ public int compareTo(Version that) {
+ if (major != that.major) {
+ return major - that.major;
+ }
+ return minor - that.minor;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other)
+ return true;
+ if (!(other instanceof Version))
+ return false;
+ return compareTo((Version) other) == 0;
+ }
+
+ @Override
+ public int hashCode() {
+ return (major << 16 + minor);
+ }
+ }
+
+ /**
+ * Lower bound binary search. Find the index to the first element in the list that compares greater than or equal to key.
+ *
+ * @param <T>
+ * Type of the input key.
+ * @param list
+ * The list
+ * @param key
+ * The input key.
+ * @param cmp
+ * Comparator for the key.
+ * @return The index to the desired element if it exists; or list.size() otherwise.
+ */
+ public static <T> int lowerBound(List<? extends T> list, T key, Comparator<? super T> cmp) {
+ int low = 0;
+ int high = list.size();
+
+ while (low < high) {
+ int mid = (low + high) >>> 1;
+ T midVal = list.get(mid);
+ int ret = cmp.compare(midVal, key);
+ if (ret < 0)
+ low = mid + 1;
+ else
+ high = mid;
+ }
+ return low;
+ }
+
+ /**
+ * Upper bound binary search. Find the index to the first element in the list that compares greater than the input key.
+ *
+ * @param <T>
+ * Type of the input key.
+ * @param list
+ * The list
+ * @param key
+ * The input key.
+ * @param cmp
+ * Comparator for the key.
+ * @return The index to the desired element if it exists; or list.size() otherwise.
+ */
+ public static <T> int upperBound(List<? extends T> list, T key, Comparator<? super T> cmp) {
+ int low = 0;
+ int high = list.size();
+
+ while (low < high) {
+ int mid = (low + high) >>> 1;
+ T midVal = list.get(mid);
+ int ret = cmp.compare(midVal, key);
+ if (ret <= 0)
+ low = mid + 1;
+ else
+ high = mid;
+ }
+ return low;
+ }
+
+ /**
+ * Lower bound binary search. Find the index to the first element in the list that compares greater than or equal to key.
+ *
+ * @param <T>
+ * Type of the input key.
+ * @param list
+ * The list
+ * @param key
+ * The input key.
+ * @return The index to the desired element if it exists; or list.size() otherwise.
+ */
+ public static <T> int lowerBound(List<? extends Comparable<? super T>> list, T key) {
+ int low = 0;
+ int high = list.size();
+
+ while (low < high) {
+ int mid = (low + high) >>> 1;
+ Comparable<? super T> midVal = list.get(mid);
+ int ret = midVal.compareTo(key);
+ if (ret < 0)
+ low = mid + 1;
+ else
+ high = mid;
+ }
+ return low;
+ }
+
+ /**
+ * Upper bound binary search. Find the index to the first element in the list that compares greater than the input key.
+ *
+ * @param <T>
+ * Type of the input key.
+ * @param list
+ * The list
+ * @param key
+ * The input key.
+ * @return The index to the desired element if it exists; or list.size() otherwise.
+ */
+ public static <T> int upperBound(List<? extends Comparable<? super T>> list, T key) {
+ int low = 0;
+ int high = list.size();
+
+ while (low < high) {
+ int mid = (low + high) >>> 1;
+ Comparable<? super T> midVal = list.get(mid);
+ int ret = midVal.compareTo(key);
+ if (ret <= 0)
+ low = mid + 1;
+ else
+ high = mid;
+ }
+ return low;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
index 54a1333,0000000..5b7b05c
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
@@@ -1,249 -1,0 +1,243 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+
+/**
+ * A Combiner that decodes each Value to type V before reducing, then encodes the result of typedReduce back to Value.
+ *
+ * Subclasses must implement a typedReduce method: public V typedReduce(Key key, Iterator<V> iter);
+ *
+ * This typedReduce method will be passed the most recent Key and an iterator over the Values (translated to Vs) for all non-deleted versions of that Key.
+ *
+ * Subclasses may implement a switch on the "type" variable to choose an Encoder in their init method.
+ */
+public abstract class TypedValueCombiner<V> extends Combiner {
+ private Encoder<V> encoder = null;
+ private boolean lossy = false;
+
+ protected static final String LOSSY = "lossy";
+
+ /**
+ * A Java Iterator that translates an Iterator<Value> to an Iterator<V> using the decode method of an Encoder.
+ */
+ private static class VIterator<V> implements Iterator<V> {
+ private Iterator<Value> source;
+ private Encoder<V> encoder;
+ private boolean lossy;
+
+ /**
+ * Constructs an Iterator<V> from an Iterator<Value>
+ *
+ * @param iter
+ * The source iterator
+ *
+ * @param encoder
+ * The Encoder whose decode method is used to translate from Value to V
+ *
+ * @param lossy
+ * Determines whether to error on failure to decode or ignore and move on
+ */
+ VIterator(Iterator<Value> iter, Encoder<V> encoder, boolean lossy) {
+ this.source = iter;
+ this.encoder = encoder;
+ this.lossy = lossy;
+ }
+
+ V next = null;
+ boolean hasNext = false;
+
+ @Override
+ public boolean hasNext() {
+ if (hasNext)
+ return true;
+
+ while (true) {
+ if (!source.hasNext())
+ return false;
+ try {
+ next = encoder.decode(source.next().get());
+ return hasNext = true;
+ } catch (ValueFormatException vfe) {
+ if (!lossy)
+ throw vfe;
+ }
+ }
+ }
+
+ @Override
+ public V next() {
+ if (!hasNext && !hasNext())
+ throw new NoSuchElementException();
+ V toRet = next;
+ next = null;
+ hasNext = false;
+ return toRet;
+ }
+
+ @Override
+ public void remove() {
+ source.remove();
+ }
+ }
+
+ /**
+ * An interface for translating from byte[] to V and back.
+ */
+ public static interface Encoder<V> {
+ public byte[] encode(V v);
+
+ public V decode(byte[] b) throws ValueFormatException;
+ }
+
+ /**
+ * Sets the Encoder<V> used to translate Values to V and back.
- *
- * @param encoder
+ */
+ protected void setEncoder(Encoder<V> encoder) {
+ this.encoder = encoder;
+ }
+
+ /**
+ * Instantiates and sets the Encoder<V> used to translate Values to V and back.
+ *
- * @param encoderClass
+ * @throws IllegalArgumentException
+ * if ClassNotFoundException, InstantiationException, or IllegalAccessException occurs
+ */
+ protected void setEncoder(String encoderClass) {
+ try {
+ @SuppressWarnings("unchecked")
+ Class<? extends Encoder<V>> clazz = (Class<? extends Encoder<V>>) AccumuloVFSClassLoader.loadClass(encoderClass, Encoder.class);
+ encoder = clazz.newInstance();
+ } catch (ClassNotFoundException e) {
+ throw new IllegalArgumentException(e);
+ } catch (InstantiationException e) {
+ throw new IllegalArgumentException(e);
+ } catch (IllegalAccessException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+
+ /**
+ * Tests whether v remains the same when encoded and decoded with the current encoder.
+ *
- * @param v
+ * @throws IllegalStateException
+ * if an encoder has not been set.
+ * @throws IllegalArgumentException
+ * if the test fails.
+ */
+ protected void testEncoder(V v) {
+ if (encoder == null)
+ throw new IllegalStateException("encoder has not been initialized");
+ testEncoder(encoder, v);
+ }
+
+ /**
+ * Tests whether v remains the same when encoded and decoded with the given encoder.
+ *
- * @param encoder
- * @param v
+ * @throws IllegalArgumentException
+ * if the test fails.
+ */
+ public static <V> void testEncoder(Encoder<V> encoder, V v) {
+ try {
+ if (!v.equals(encoder.decode(encoder.encode(v))))
+ throw new IllegalArgumentException("something wrong with " + encoder.getClass().getName() + " -- doesn't encode and decode " + v + " properly");
+ } catch (ClassCastException e) {
+ throw new IllegalArgumentException(encoder.getClass().getName() + " doesn't encode " + v.getClass().getName());
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ TypedValueCombiner<V> newInstance = (TypedValueCombiner<V>) super.deepCopy(env);
+ newInstance.setEncoder(encoder);
+ return newInstance;
+ }
+
+ @Override
+ public Value reduce(Key key, Iterator<Value> iter) {
+ return new Value(encoder.encode(typedReduce(key, new VIterator<V>(iter, encoder, lossy))));
+ }
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ super.init(source, options, env);
+ setLossyness(options);
+ }
+
+ private void setLossyness(Map<String,String> options) {
+ String loss = options.get(LOSSY);
+ if (loss == null)
+ lossy = false;
+ else
+ lossy = Boolean.parseBoolean(loss);
+ }
+
+ @Override
+ public IteratorOptions describeOptions() {
+ IteratorOptions io = super.describeOptions();
+ io.addNamedOption(LOSSY, "if true, failed decodes are ignored. Otherwise combiner will error on failed decodes (default false): <TRUE|FALSE>");
+ return io;
+ }
+
+ @Override
+ public boolean validateOptions(Map<String,String> options) {
+ if (super.validateOptions(options) == false)
+ return false;
+ try {
+ setLossyness(options);
+ } catch (Exception e) {
+ throw new IllegalArgumentException("bad boolean " + LOSSY + ":" + options.get(LOSSY));
+ }
+ return true;
+ }
+
+ /**
+ * A convenience method to set the "lossy" option on a TypedValueCombiner. If true, the combiner will ignore any values which fail to decode. Otherwise, the
+ * combiner will throw an error which will interrupt the action (and prevent potential data loss). False is the default behavior.
+ *
+ * @param is
+ * iterator settings object to configure
+ * @param lossy
+ * if true the combiner will ignored values which fail to decode; otherwise error.
+ */
+ public static void setLossyness(IteratorSetting is, boolean lossy) {
+ is.addOption(LOSSY, Boolean.toString(lossy));
+ }
+
+ /**
+ * Reduces a list of V into a single V.
+ *
+ * @param key
+ * The most recent version of the Key being reduced.
+ *
+ * @param iter
+ * An iterator over the V for different versions of the key.
+ *
+ * @return The combined V.
+ */
+ public abstract V typedReduce(Key key, Iterator<V> iter);
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/ValueFormatException.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/ValueFormatException.java
index 7bb2228,0000000..7ede7fe
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/ValueFormatException.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/ValueFormatException.java
@@@ -1,40 -1,0 +1,34 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators;
+
+/**
+ * Exception used for TypedValueCombiner and it's Encoders decode() function
+ */
+public class ValueFormatException extends IllegalArgumentException {
+
- /**
- * @param string
- */
+ public ValueFormatException(String string) {
+ super(string);
+ }
+
- /**
- * @param nfe
- */
+ public ValueFormatException(Exception nfe) {
+ super(nfe);
+ }
+
+ private static final long serialVersionUID = 4170291568272971821L;
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/system/MapFileIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/system/MapFileIterator.java
index e5fe62a,0000000..37a234c
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/system/MapFileIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/system/MapFileIterator.java
@@@ -1,162 -1,0 +1,156 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators.system;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileSKVIterator;
+import org.apache.accumulo.core.file.NoSuchMetaStoreException;
+import org.apache.accumulo.core.file.map.MapFileUtil;
+import org.apache.accumulo.core.iterators.IterationInterruptedException;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.MapFile.Reader;
+import org.apache.log4j.Logger;
+
+public class MapFileIterator implements FileSKVIterator {
+ private static final Logger log = Logger.getLogger(MapFileIterator.class);
+
+ private Reader reader;
+ private Value topValue;
+ private Key topKey;
+ private AtomicBoolean interruptFlag;
+ private int interruptCheckCount = 0;
+ private FileSystem fs;
+ private String dirName;
+
- /**
- * @param acuconf
- * @param fs
- * @param dir
- * @param conf
- * @throws IOException
- */
+ public MapFileIterator(AccumuloConfiguration acuconf, FileSystem fs, String dir, Configuration conf) throws IOException {
+ this.reader = MapFileUtil.openMapFile(acuconf, fs, dir, conf);
+ this.fs = fs;
+ this.dirName = dir;
+ }
+
+ @Override
+ public void setInterruptFlag(AtomicBoolean flag) {
+ this.interruptFlag = flag;
+ }
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean hasTop() {
+ return topKey != null;
+ }
+
+ @Override
+ public void next() throws IOException {
+ if (interruptFlag != null && interruptCheckCount++ % 100 == 0 && interruptFlag.get())
+ throw new IterationInterruptedException();
+
+ reader.next(topKey, topValue);
+ }
+
++ @Override
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
+ if (columnFamilies.size() != 0 || inclusive) {
+ throw new IllegalArgumentException("I do not know how to filter column families");
+ }
+
+ if (range == null)
+ throw new IllegalArgumentException("Cannot seek to null range");
+
+ if (interruptFlag != null && interruptFlag.get())
+ throw new IterationInterruptedException();
+
+ Key key = range.getStartKey();
+ if (key == null) {
+ key = new Key();
+ }
+
+ reader.seek(key);
+
+ while (hasTop() && range.beforeStartKey(getTopKey())) {
+ next();
+ }
+ }
+
+ @Override
+ public Key getTopKey() {
+ return topKey;
+ }
+
+ @Override
+ public Value getTopValue() {
+ return topValue;
+ }
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ try {
+ SortedKeyValueIterator<Key,Value> other = env.reserveMapFileReader(dirName);
+ ((InterruptibleIterator) other).setInterruptFlag(interruptFlag);
+ log.debug("deep copying MapFile: " + this + " -> " + other);
+ return other;
+ } catch (IOException e) {
+ log.error("failed to clone map file reader", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public Key getFirstKey() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Key getLastKey() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DataInputStream getMetaStore(String name) throws IOException {
+ Path path = new Path(this.dirName, name);
+ if (!fs.exists(path))
+ throw new NoSuchMetaStoreException("name = " + name);
+ return fs.open(path);
+ }
+
+ @Override
+ public void closeDeepCopies() throws IOException {
+ // nothing to do, deep copies are externally managed/closed
+ }
+
+ @Override
+ public void close() throws IOException {
+ reader.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
index 4f8207c,0000000..86798dd
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
@@@ -1,104 -1,0 +1,101 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators.user;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Map;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.Filter;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+
+/**
+ * This iterator provides exact string matching. It searches both the Key and Value for the string. The string to match is specified by the "term" option.
+ */
+public class GrepIterator extends Filter {
+
+ private byte term[];
+
+ @Override
+ public boolean accept(Key k, Value v) {
+ return match(v.get()) || match(k.getRowData()) || match(k.getColumnFamilyData()) || match(k.getColumnQualifierData());
+ }
+
+ private boolean match(ByteSequence bs) {
+ return indexOf(bs.getBackingArray(), bs.offset(), bs.length(), term) >= 0;
+ }
+
+ private boolean match(byte[] ba) {
+ return indexOf(ba, 0, ba.length, term) >= 0;
+ }
+
+ // copied code below from java string and modified
+
+ private static int indexOf(byte[] source, int sourceOffset, int sourceCount, byte[] target) {
+ byte first = target[0];
+ int targetCount = target.length;
+ int max = sourceOffset + (sourceCount - targetCount);
+
+ for (int i = sourceOffset; i <= max; i++) {
+ /* Look for first character. */
+ if (source[i] != first) {
+ while (++i <= max && source[i] != first)
+ continue;
+ }
+
+ /* Found first character, now look at the rest of v2 */
+ if (i <= max) {
+ int j = i + 1;
+ int end = j + targetCount - 1;
+ for (int k = 1; j < end && source[j] == target[k]; j++, k++)
+ continue;
+
+ if (j == end) {
+ /* Found whole string. */
+ return i - sourceOffset;
+ }
+ }
+ }
+ return -1;
+ }
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ GrepIterator copy = (GrepIterator) super.deepCopy(env);
+ copy.term = Arrays.copyOf(term, term.length);
+ return copy;
+ }
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ super.init(source, options, env);
+ term = options.get("term").getBytes(Constants.UTF8);
+ }
+
+ /**
+ * Encode the grep term as an option for a ScanIterator
- *
- * @param cfg
- * @param term
+ */
+ public static void setTerm(IteratorSetting cfg, String term) {
+ cfg.addOption("term", term);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
index 447200b,0000000..39cba6d
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
@@@ -1,558 -1,0 +1,548 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators.user;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+/**
+ * This iterator facilitates document-partitioned indexing. It involves grouping a set of documents together and indexing those documents into a single row of
+ * an Accumulo table. This allows a tablet server to perform boolean AND operations on terms in the index.
+ *
+ * The table structure should have the following form:
+ *
+ * row: shardID, colfam: term, colqual: docID
+ *
+ * When you configure this iterator with a set of terms (column families), it will return only the docIDs that appear with all of the specified terms. The
+ * result will have an empty column family, as follows:
+ *
+ * row: shardID, colfam: (empty), colqual: docID
+ *
+ * This iterator is commonly used with BatchScanner or AccumuloInputFormat, to parallelize the search over all shardIDs.
+ *
+ * This iterator will *ignore* any columnFamilies passed to {@link #seek(Range, Collection, boolean)} as it performs intersections over terms. Extending classes
+ * should override the {@link TermSource#seekColfams} in their implementation's {@link #init(SortedKeyValueIterator, Map, IteratorEnvironment)} method.
+ *
+ * README.shard in docs/examples shows an example of using the IntersectingIterator.
+ */
+public class IntersectingIterator implements SortedKeyValueIterator<Key,Value> {
+
+ protected Text nullText = new Text();
+
+ protected Text getPartition(Key key) {
+ return key.getRow();
+ }
+
+ protected Text getTerm(Key key) {
+ return key.getColumnFamily();
+ }
+
+ protected Text getDocID(Key key) {
+ return key.getColumnQualifier();
+ }
+
+ protected Key buildKey(Text partition, Text term) {
+ return new Key(partition, (term == null) ? nullText : term);
+ }
+
+ protected Key buildKey(Text partition, Text term, Text docID) {
+ return new Key(partition, (term == null) ? nullText : term, docID);
+ }
+
+ protected Key buildFollowingPartitionKey(Key key) {
+ return key.followingKey(PartialKey.ROW);
+ }
+
+ protected static final Logger log = Logger.getLogger(IntersectingIterator.class);
+
+ public static class TermSource {
+ public SortedKeyValueIterator<Key,Value> iter;
+ public Text term;
+ public Collection<ByteSequence> seekColfams;
+ public boolean notFlag;
+
+ public TermSource(TermSource other) {
+ this.iter = other.iter;
+ this.term = other.term;
+ this.notFlag = other.notFlag;
+ this.seekColfams = other.seekColfams;
+ }
+
+ public TermSource(SortedKeyValueIterator<Key,Value> iter, Text term) {
+ this(iter, term, false);
+ }
+
+ public TermSource(SortedKeyValueIterator<Key,Value> iter, Text term, boolean notFlag) {
+ this.iter = iter;
+ this.term = term;
+ this.notFlag = notFlag;
+ // The desired column families for this source is the term itself
+ this.seekColfams = Collections.<ByteSequence> singletonList(new ArrayByteSequence(term.getBytes(), 0, term.getLength()));
+ }
+
+ public String getTermString() {
+ return (this.term == null) ? "Iterator" : this.term.toString();
+ }
+ }
+
+ protected TermSource[] sources;
+ int sourcesCount = 0;
+
+ Range overallRange;
+
+ // query-time settings
+ protected Text currentPartition = null;
+ protected Text currentDocID = new Text(emptyByteArray);
+ static final byte[] emptyByteArray = new byte[0];
+
+ protected Key topKey = null;
+ protected Value value = new Value(emptyByteArray);
+
+ public IntersectingIterator() {}
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ return new IntersectingIterator(this, env);
+ }
+
+ private IntersectingIterator(IntersectingIterator other, IteratorEnvironment env) {
+ if (other.sources != null) {
+ sourcesCount = other.sourcesCount;
+ sources = new TermSource[sourcesCount];
+ for (int i = 0; i < sourcesCount; i++) {
+ sources[i] = new TermSource(other.sources[i].iter.deepCopy(env), other.sources[i].term);
+ }
+ }
+ }
+
+ @Override
+ public Key getTopKey() {
+ return topKey;
+ }
+
+ @Override
+ public Value getTopValue() {
+ // we don't really care about values
+ return value;
+ }
+
+ @Override
+ public boolean hasTop() {
+ return currentPartition != null;
+ }
+
+ // precondition: currentRow is not null
+ private boolean seekOneSource(int sourceID) throws IOException {
+ // find the next key in the appropriate column family that is at or beyond the cursor (currentRow, currentCQ)
+ // advance the cursor if this source goes beyond it
+ // return whether we advanced the cursor
+
+ // within this loop progress must be made in one of the following forms:
+ // - currentRow or currentCQ must be increased
+ // - the given source must advance its iterator
+ // this loop will end when any of the following criteria are met
+ // - the iterator for the given source is pointing to the key (currentRow, columnFamilies[sourceID], currentCQ)
+ // - the given source is out of data and currentRow is set to null
+ // - the given source has advanced beyond the endRow and currentRow is set to null
+ boolean advancedCursor = false;
+
+ if (sources[sourceID].notFlag) {
+ while (true) {
+ if (sources[sourceID].iter.hasTop() == false) {
+ // an empty column that you are negating is a valid condition
+ break;
+ }
+ // check if we're past the end key
+ int endCompare = -1;
+ // we should compare the row to the end of the range
+ if (overallRange.getEndKey() != null) {
+ endCompare = overallRange.getEndKey().getRow().compareTo(sources[sourceID].iter.getTopKey().getRow());
+ if ((!overallRange.isEndKeyInclusive() && endCompare <= 0) || endCompare < 0) {
+ // an empty column that you are negating is a valid condition
+ break;
+ }
+ }
+ int partitionCompare = currentPartition.compareTo(getPartition(sources[sourceID].iter.getTopKey()));
+ // check if this source is already at or beyond currentRow
+ // if not, then seek to at least the current row
+
+ if (partitionCompare > 0) {
+ // seek to at least the currentRow
+ Key seekKey = buildKey(currentPartition, sources[sourceID].term);
+ sources[sourceID].iter.seek(new Range(seekKey, true, null, false), sources[sourceID].seekColfams, true);
+ continue;
+ }
+ // check if this source has gone beyond currentRow
+ // if so, this is a valid condition for negation
+ if (partitionCompare < 0) {
+ break;
+ }
+ // we have verified that the current source is positioned in currentRow
+ // now we must make sure we're in the right columnFamily in the current row
+ // Note: Iterators are auto-magically set to the correct columnFamily
+ if (sources[sourceID].term != null) {
+ int termCompare = sources[sourceID].term.compareTo(getTerm(sources[sourceID].iter.getTopKey()));
+ // check if this source is already on the right columnFamily
+ // if not, then seek forwards to the right columnFamily
+ if (termCompare > 0) {
+ Key seekKey = buildKey(currentPartition, sources[sourceID].term, currentDocID);
+ sources[sourceID].iter.seek(new Range(seekKey, true, null, false), sources[sourceID].seekColfams, true);
+ continue;
+ }
+ // check if this source is beyond the right columnFamily
+ // if so, then this is a valid condition for negating
+ if (termCompare < 0) {
+ break;
+ }
+ }
+
+ // we have verified that we are in currentRow and the correct column family
+ // make sure we are at or beyond columnQualifier
+ Text docID = getDocID(sources[sourceID].iter.getTopKey());
+ int docIDCompare = currentDocID.compareTo(docID);
+ // If we are past the target, this is a valid result
+ if (docIDCompare < 0) {
+ break;
+ }
+ // if this source is not yet at the currentCQ then advance in this source
+ if (docIDCompare > 0) {
+ // seek forwards
+ Key seekKey = buildKey(currentPartition, sources[sourceID].term, currentDocID);
+ sources[sourceID].iter.seek(new Range(seekKey, true, null, false), sources[sourceID].seekColfams, true);
+ continue;
+ }
+ // if we are equal to the target, this is an invalid result.
+ // Force the entire process to go to the next row.
+ // We are advancing column 0 because we forced that column to not contain a !
+ // when we did the init()
+ if (docIDCompare == 0) {
+ sources[0].iter.next();
+ advancedCursor = true;
+ break;
+ }
+ }
+ } else {
+ while (true) {
+ if (sources[sourceID].iter.hasTop() == false) {
+ currentPartition = null;
+ // setting currentRow to null counts as advancing the cursor
+ return true;
+ }
+ // check if we're past the end key
+ int endCompare = -1;
+ // we should compare the row to the end of the range
+
+ if (overallRange.getEndKey() != null) {
+ endCompare = overallRange.getEndKey().getRow().compareTo(sources[sourceID].iter.getTopKey().getRow());
+ if ((!overallRange.isEndKeyInclusive() && endCompare <= 0) || endCompare < 0) {
+ currentPartition = null;
+ // setting currentRow to null counts as advancing the cursor
+ return true;
+ }
+ }
+ int partitionCompare = currentPartition.compareTo(getPartition(sources[sourceID].iter.getTopKey()));
+ // check if this source is already at or beyond currentRow
+ // if not, then seek to at least the current row
+ if (partitionCompare > 0) {
+ // seek to at least the currentRow
+ Key seekKey = buildKey(currentPartition, sources[sourceID].term);
+ sources[sourceID].iter.seek(new Range(seekKey, true, null, false), sources[sourceID].seekColfams, true);
+ continue;
+ }
+ // check if this source has gone beyond currentRow
+ // if so, advance currentRow
+ if (partitionCompare < 0) {
+ currentPartition.set(getPartition(sources[sourceID].iter.getTopKey()));
+ currentDocID.set(emptyByteArray);
+ advancedCursor = true;
+ continue;
+ }
+ // we have verified that the current source is positioned in currentRow
+ // now we must make sure we're in the right columnFamily in the current row
+ // Note: Iterators are auto-magically set to the correct columnFamily
+
+ if (sources[sourceID].term != null) {
+ int termCompare = sources[sourceID].term.compareTo(getTerm(sources[sourceID].iter.getTopKey()));
+ // check if this source is already on the right columnFamily
+ // if not, then seek forwards to the right columnFamily
+ if (termCompare > 0) {
+ Key seekKey = buildKey(currentPartition, sources[sourceID].term, currentDocID);
+ sources[sourceID].iter.seek(new Range(seekKey, true, null, false), sources[sourceID].seekColfams, true);
+ continue;
+ }
+ // check if this source is beyond the right columnFamily
+ // if so, then seek to the next row
+ if (termCompare < 0) {
+ // we're out of entries in the current row, so seek to the next one
+ // byte[] currentRowBytes = currentRow.getBytes();
+ // byte[] nextRow = new byte[currentRowBytes.length + 1];
+ // System.arraycopy(currentRowBytes, 0, nextRow, 0, currentRowBytes.length);
+ // nextRow[currentRowBytes.length] = (byte)0;
+ // // we should reuse text objects here
+ // sources[sourceID].seek(new Key(new Text(nextRow),columnFamilies[sourceID]));
+ if (endCompare == 0) {
+ // we're done
+ currentPartition = null;
+ // setting currentRow to null counts as advancing the cursor
+ return true;
+ }
+ Key seekKey = buildFollowingPartitionKey(sources[sourceID].iter.getTopKey());
+ sources[sourceID].iter.seek(new Range(seekKey, true, null, false), sources[sourceID].seekColfams, true);
+ continue;
+ }
+ }
+ // we have verified that we are in currentRow and the correct column family
+ // make sure we are at or beyond columnQualifier
+ Text docID = getDocID(sources[sourceID].iter.getTopKey());
+ int docIDCompare = currentDocID.compareTo(docID);
+ // if this source has advanced beyond the current column qualifier then advance currentCQ and return true
+ if (docIDCompare < 0) {
+ currentDocID.set(docID);
+ advancedCursor = true;
+ break;
+ }
+ // if this source is not yet at the currentCQ then seek in this source
+ if (docIDCompare > 0) {
+ // seek forwards
+ Key seekKey = buildKey(currentPartition, sources[sourceID].term, currentDocID);
+ sources[sourceID].iter.seek(new Range(seekKey, true, null, false), sources[sourceID].seekColfams, true);
+ continue;
+ }
+ // this source is at the current row, in its column family, and at currentCQ
+ break;
+ }
+ }
+ return advancedCursor;
+ }
+
+ @Override
+ public void next() throws IOException {
+ if (currentPartition == null) {
+ return;
+ }
+ // precondition: the current row is set up and the sources all have the same column qualifier
+ // while we don't have a match, seek in the source with the smallest column qualifier
+ sources[0].iter.next();
+ advanceToIntersection();
+ }
+
+ protected void advanceToIntersection() throws IOException {
+ boolean cursorChanged = true;
+ while (cursorChanged) {
+ // seek all of the sources to at least the highest seen column qualifier in the current row
+ cursorChanged = false;
+ for (int i = 0; i < sourcesCount; i++) {
+ if (currentPartition == null) {
+ topKey = null;
+ return;
+ }
+ if (seekOneSource(i)) {
+ cursorChanged = true;
+ break;
+ }
+ }
+ }
+ topKey = buildKey(currentPartition, nullText, currentDocID);
+ }
+
+ public static String stringTopKey(SortedKeyValueIterator<Key,Value> iter) {
+ if (iter.hasTop())
+ return iter.getTopKey().toString();
+ return "";
+ }
+
+ private static final String columnFamiliesOptionName = "columnFamilies";
+ private static final String notFlagOptionName = "notFlag";
+
+ /**
- * @param columns
+ * @return encoded columns
+ */
+ protected static String encodeColumns(Text[] columns) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < columns.length; i++) {
+ sb.append(new String(Base64.encodeBase64(TextUtil.getBytes(columns[i])), Constants.UTF8));
+ sb.append('\n');
+ }
+ return sb.toString();
+ }
+
+ /**
- * @param flags
+ * @return encoded flags
+ */
+ protected static String encodeBooleans(boolean[] flags) {
+ byte[] bytes = new byte[flags.length];
+ for (int i = 0; i < flags.length; i++) {
+ if (flags[i])
+ bytes[i] = 1;
+ else
+ bytes[i] = 0;
+ }
+ return new String(Base64.encodeBase64(bytes), Constants.UTF8);
+ }
+
+ protected static Text[] decodeColumns(String columns) {
+ String[] columnStrings = columns.split("\n");
+ Text[] columnTexts = new Text[columnStrings.length];
+ for (int i = 0; i < columnStrings.length; i++) {
+ columnTexts[i] = new Text(Base64.decodeBase64(columnStrings[i].getBytes(Constants.UTF8)));
+ }
+ return columnTexts;
+ }
+
+ /**
- * @param flags
+ * @return decoded flags
+ */
+ protected static boolean[] decodeBooleans(String flags) {
+ // return null of there were no flags
+ if (flags == null)
+ return null;
+
+ byte[] bytes = Base64.decodeBase64(flags.getBytes(Constants.UTF8));
+ boolean[] bFlags = new boolean[bytes.length];
+ for (int i = 0; i < bytes.length; i++) {
+ if (bytes[i] == 1)
+ bFlags[i] = true;
+ else
+ bFlags[i] = false;
+ }
+ return bFlags;
+ }
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ Text[] terms = decodeColumns(options.get(columnFamiliesOptionName));
+ boolean[] notFlag = decodeBooleans(options.get(notFlagOptionName));
+
+ if (terms.length < 2) {
+ throw new IllegalArgumentException("IntersectionIterator requires two or more columns families");
+ }
+
+ // Scan the not flags.
+ // There must be at least one term that isn't negated
+ // And we are going to re-order such that the first term is not a ! term
+ if (notFlag == null) {
+ notFlag = new boolean[terms.length];
+ for (int i = 0; i < terms.length; i++)
+ notFlag[i] = false;
+ }
+ if (notFlag[0]) {
+ for (int i = 1; i < notFlag.length; i++) {
+ if (notFlag[i] == false) {
+ Text swapFamily = new Text(terms[0]);
+ terms[0].set(terms[i]);
+ terms[i].set(swapFamily);
+ notFlag[0] = false;
+ notFlag[i] = true;
+ break;
+ }
+ }
+ if (notFlag[0]) {
+ throw new IllegalArgumentException("IntersectionIterator requires at lest one column family without not");
+ }
+ }
+
+ sources = new TermSource[terms.length];
+ sources[0] = new TermSource(source, terms[0]);
+ for (int i = 1; i < terms.length; i++) {
+ sources[i] = new TermSource(source.deepCopy(env), terms[i], notFlag[i]);
+ }
+ sourcesCount = terms.length;
+ }
+
+ @Override
+ public void seek(Range range, Collection<ByteSequence> seekColumnFamilies, boolean inclusive) throws IOException {
+ overallRange = new Range(range);
+ currentPartition = new Text();
+ currentDocID.set(emptyByteArray);
+
+ // seek each of the sources to the right column family within the row given by key
+ for (int i = 0; i < sourcesCount; i++) {
+ Key sourceKey;
+ if (range.getStartKey() != null) {
+ if (range.getStartKey().getColumnQualifier() != null) {
+ sourceKey = buildKey(getPartition(range.getStartKey()), sources[i].term, range.getStartKey().getColumnQualifier());
+ } else {
+ sourceKey = buildKey(getPartition(range.getStartKey()), sources[i].term);
+ }
+ // Seek only to the term for this source as a column family
+ sources[i].iter.seek(new Range(sourceKey, true, null, false), sources[i].seekColfams, true);
+ } else {
+ // Seek only to the term for this source as a column family
+ sources[i].iter.seek(range, sources[i].seekColfams, true);
+ }
+ }
+ advanceToIntersection();
+ }
+
+ public void addSource(SortedKeyValueIterator<Key,Value> source, IteratorEnvironment env, Text term, boolean notFlag) {
+ // Check if we have space for the added Source
+ if (sources == null) {
+ sources = new TermSource[1];
+ } else {
+ // allocate space for node, and copy current tree.
+ // TODO: Should we change this to an ArrayList so that we can just add() ? - ACCUMULO-1309
+ TermSource[] localSources = new TermSource[sources.length + 1];
+ int currSource = 0;
+ for (TermSource myTerm : sources) {
+ // TODO: Do I need to call new here? or can I just re-use the term? - ACCUMULO-1309
+ localSources[currSource] = new TermSource(myTerm);
+ currSource++;
+ }
+ sources = localSources;
+ }
+ sources[sourcesCount] = new TermSource(source.deepCopy(env), term, notFlag);
+ sourcesCount++;
+ }
+
+ /**
+ * Encode the columns to be used when iterating.
- *
- * @param cfg
- * @param columns
+ */
+ public static void setColumnFamilies(IteratorSetting cfg, Text[] columns) {
+ if (columns.length < 2)
+ throw new IllegalArgumentException("Must supply at least two terms to intersect");
+ cfg.addOption(IntersectingIterator.columnFamiliesOptionName, IntersectingIterator.encodeColumns(columns));
+ }
+
+ /**
+ * Encode columns and NOT flags indicating which columns should be negated (docIDs will be excluded if matching negated columns, instead of included).
- *
- * @param cfg
- * @param columns
- * @param notFlags
+ */
+ public static void setColumnFamilies(IteratorSetting cfg, Text[] columns, boolean[] notFlags) {
+ if (columns.length < 2)
+ throw new IllegalArgumentException("Must supply at least two terms to intersect");
+ if (columns.length != notFlags.length)
+ throw new IllegalArgumentException("columns and notFlags arrays must be the same length");
+ setColumnFamilies(cfg, columns);
+ cfg.addOption(IntersectingIterator.notFlagOptionName, IntersectingIterator.encodeBooleans(notFlags));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/user/RowFilter.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/RowFilter.java
index a232796,0000000..2d2fa74
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/RowFilter.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/RowFilter.java
@@@ -1,165 -1,0 +1,164 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators.user;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.WrappingIterator;
+import org.apache.hadoop.io.Text;
+
+/**
+ * This iterator makes it easy to select rows that meet a given criteria. Its an alternative to the {@link WholeRowIterator}. There are a few things to consider
+ * when deciding which one to use.
+ *
+ * First the WholeRowIterator requires that the row fit in memory and that the entire row is read before a decision is made. This iterator has neither
+ * requirement, it allows seeking within a row to avoid reading the entire row to make a decision. So even if your rows fit into memory, this extending this
+ * iterator may be better choice because you can seek.
+ *
+ * Second the WholeRowIterator is currently the only way to achieve row isolation with the {@link BatchScanner}. With the normal {@link Scanner} row isolation
+ * can be enabled and this Iterator may be used.
+ *
+ * Third the row acceptance test will be executed every time this Iterator is seeked. If the row is large, then the row will fetched in batches of key/values.
+ * As each batch is fetched the test may be re-executed because the iterator stack is reseeked for each batch. The batch size may be increased to reduce the
+ * number of times the test is executed. With the normal Scanner, if isolation is enabled then it will read an entire row w/o seeking this iterator.
+ *
+ */
+public abstract class RowFilter extends WrappingIterator {
+
+ private RowIterator decisionIterator;
+ private Collection<ByteSequence> columnFamilies;
+ Text currentRow;
+ private boolean inclusive;
+ private Range range;
+ private boolean hasTop;
+
+ private static class RowIterator extends WrappingIterator {
+ private Range rowRange;
+ private boolean hasTop;
+
+ RowIterator(SortedKeyValueIterator<Key,Value> source) {
+ super.setSource(source);
+ }
+
+ void setRow(Range row) {
+ this.rowRange = row;
+ }
+
+ @Override
+ public boolean hasTop() {
+ return hasTop && super.hasTop();
+ }
+
+ @Override
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
+
+ range = rowRange.clip(range, true);
+ if (range == null) {
+ hasTop = false;
+ } else {
+ hasTop = true;
+ super.seek(range, columnFamilies, inclusive);
+ }
+ }
+ }
+
+ private void skipRows() throws IOException {
+ SortedKeyValueIterator<Key,Value> source = getSource();
+ while (source.hasTop()) {
+ Text row = source.getTopKey().getRow();
+
+ if (currentRow != null && currentRow.equals(row))
+ break;
+
+ Range rowRange = new Range(row);
+ decisionIterator.setRow(rowRange);
+ decisionIterator.seek(rowRange, columnFamilies, inclusive);
+
+ if (acceptRow(decisionIterator)) {
+ currentRow = row;
+ break;
+ } else {
+ currentRow = null;
+ int count = 0;
+ while (source.hasTop() && count < 10 && source.getTopKey().getRow().equals(row)) {
+ count++;
+ source.next();
+ }
+
+ if (source.hasTop() && source.getTopKey().getRow().equals(row)) {
+ Range nextRow = new Range(row, false, null, false);
+ nextRow = range.clip(nextRow, true);
+ if (nextRow == null)
+ hasTop = false;
+ else
+ source.seek(nextRow, columnFamilies, inclusive);
+ }
+ }
+ }
+ }
+
+ /**
+ * Implementation should return false to suppress a row.
+ *
+ *
+ * @param rowIterator
+ * - An iterator over the row. This iterator is confined to the row. Seeking past the end of the row will return no data. Seeking before the row will
+ * always set top to the first column in the current row. By default this iterator will only see the columns the parent was seeked with. To see more
+ * columns reseek this iterator with those columns.
+ * @return false if a row should be suppressed, otherwise true.
- * @throws IOException
+ */
+ public abstract boolean acceptRow(SortedKeyValueIterator<Key,Value> rowIterator) throws IOException;
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ super.init(source, options, env);
+ this.decisionIterator = new RowIterator(source.deepCopy(env));
+ }
+
+ @Override
+ public boolean hasTop() {
+ return hasTop && super.hasTop();
+ }
+
+ @Override
+ public void next() throws IOException {
+ super.next();
+ skipRows();
+ }
+
+ @Override
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
+ super.seek(range, columnFamilies, inclusive);
+ this.columnFamilies = columnFamilies;
+ this.inclusive = inclusive;
+ this.range = range;
+ currentRow = null;
+ hasTop = true;
+ skipRows();
+
+ }
+}
[11/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/data/Range.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Range.java
index 65873c3,0000000..122436b
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/data/Range.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Range.java
@@@ -1,906 -1,0 +1,899 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
+import java.io.DataInput;
+import java.io.DataOutput;
- import java.io.InvalidObjectException;
+import java.io.IOException;
++import java.io.InvalidObjectException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.accumulo.core.data.thrift.TRange;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+
+/**
+ * This class is used to specify a range of Accumulo Keys.
+ *
+ */
+
+public class Range implements WritableComparable<Range> {
+
+ private Key start;
+ private Key stop;
+ private boolean startKeyInclusive;
+ private boolean stopKeyInclusive;
+ private boolean infiniteStartKey;
+ private boolean infiniteStopKey;
+
+ /**
+ * Creates a range that goes from negative to positive infinity
+ */
+
+ public Range() {
+ this((Key) null, true, (Key) null, true);
+ }
+
+ /**
+ * Creates a range from startKey inclusive to endKey inclusive
+ *
+ * @param startKey
+ * set this to null when negative infinity is needed
+ * @param endKey
+ * set this to null when positive infinity is needed
+ */
+ public Range(Key startKey, Key endKey) {
+ this(startKey, true, endKey, true);
+ }
+
+ /**
+ * Creates a range that covers an entire row
+ *
+ * @param row
+ * set this to null to cover all rows
+ */
+ public Range(CharSequence row) {
+ this(row, true, row, true);
+ }
+
+ /**
+ * Creates a range that covers an entire row
+ *
+ * @param row
+ * set this to null to cover all rows
+ */
+ public Range(Text row) {
+ this(row, true, row, true);
+ }
+
+ /**
+ * Creates a range from startRow inclusive to endRow inclusive
+ *
+ * @param startRow
+ * set this to null when negative infinity is needed
+ * @param endRow
+ * set this to null when positive infinity is needed
+ */
+ public Range(Text startRow, Text endRow) {
+ this(startRow, true, endRow, true);
+ }
+
+ /**
+ * Creates a range from startRow inclusive to endRow inclusive
+ *
+ * @param startRow
+ * set this to null when negative infinity is needed
+ * @param endRow
+ * set this to null when positive infinity is needed
+ */
+ public Range(CharSequence startRow, CharSequence endRow) {
+ this(startRow, true, endRow, true);
+ }
+
+ /**
+ * Creates a range from startRow to endRow
+ *
+ * @param startRow
+ * set this to null when negative infinity is needed
+ * @param startRowInclusive
+ * determines if the start row is skipped
+ * @param endRow
+ * set this to null when positive infinity is needed
+ * @param endRowInclusive
+ * determines if the endRow is included
+ */
+
+ public Range(Text startRow, boolean startRowInclusive, Text endRow, boolean endRowInclusive) {
+ this((startRow == null ? null : (startRowInclusive ? new Key(startRow) : new Key(startRow).followingKey(PartialKey.ROW))), true, (endRow == null ? null
+ : (endRowInclusive ? new Key(endRow).followingKey(PartialKey.ROW) : new Key(endRow))), false);
+ }
+
+ /**
+ * Creates a range from startRow to endRow
+ *
+ * @param startRow
+ * set this to null when negative infinity is needed
+ * @param startRowInclusive
+ * determines if the start row is skipped
+ * @param endRow
+ * set this to null when positive infinity is needed
+ * @param endRowInclusive
+ * determines if the endRow is included
+ */
+
+ public Range(CharSequence startRow, boolean startRowInclusive, CharSequence endRow, boolean endRowInclusive) {
+ this(startRow == null ? null : new Text(startRow.toString()), startRowInclusive, endRow == null ? null : new Text(endRow.toString()), endRowInclusive);
+ }
+
+ /**
+ * Creates a range from startKey to endKey
+ *
+ * @param startKey
+ * set this to null when negative infinity is needed
+ * @param startKeyInclusive
+ * determines if the ranges includes the start key
+ * @param endKey
+ * set this to null when infinity is needed
+ * @param endKeyInclusive
+ * determines if the range includes the end key
+ */
+ public Range(Key startKey, boolean startKeyInclusive, Key endKey, boolean endKeyInclusive) {
+ this.start = startKey;
+ this.startKeyInclusive = startKeyInclusive;
+ this.infiniteStartKey = startKey == null;
+ this.stop = endKey;
+ this.stopKeyInclusive = endKeyInclusive;
+ this.infiniteStopKey = stop == null;
+
+ if (!infiniteStartKey && !infiniteStopKey && beforeStartKey(endKey)) {
+ throw new IllegalArgumentException("Start key must be less than end key in range (" + startKey + ", " + endKey + ")");
+ }
+ }
+
+ /**
+ * Copies a range
+ */
+ public Range(Range range) {
+ this(range.start, range.startKeyInclusive, range.infiniteStartKey, range.stop, range.stopKeyInclusive, range.infiniteStopKey);
+ }
+
+ /**
+ * Creates a range from start to stop.
+ *
+ * @param start
+ * set this to null when negative infinity is needed
+ * @param stop
+ * set this to null when infinity is needed
+ * @param startKeyInclusive
+ * determines if the ranges includes the start key
+ * @param stopKeyInclusive
+ * determines if the range includes the end key
+ * @param infiniteStartKey
+ * true if start key is negative infinity (null)
+ * @param infiniteStopKey
+ * true if stop key is positive infinity (null)
+ * @throws IllegalArgumentException if stop is before start, or infiniteStartKey is true but start is not null, or infiniteStopKey is true but stop is not
+ * null
+ */
+ public Range(Key start, Key stop, boolean startKeyInclusive, boolean stopKeyInclusive, boolean infiniteStartKey, boolean infiniteStopKey) {
+ this(start, startKeyInclusive, infiniteStartKey, stop, stopKeyInclusive, infiniteStopKey);
+ if (!infiniteStartKey && !infiniteStopKey && beforeStartKey(stop)) {
+ throw new IllegalArgumentException("Start key must be less than end key in range (" + start + ", " + stop + ")");
+ }
+ }
+
+ /**
+ * Creates a range from start to stop. Unlike the public six-argument method,
+ * this one does not assure that stop is after start, which helps performance
+ * in cases where that assurance is already in place.
+ *
+ * @param start
+ * set this to null when negative infinity is needed
+ * @param startKeyInclusive
+ * determines if the ranges includes the start key
+ * @param infiniteStartKey
+ * true if start key is negative infinity (null)
+ * @param stop
+ * set this to null when infinity is needed
+ * @param stopKeyInclusive
+ * determines if the range includes the end key
+ * @param infiniteStopKey
+ * true if stop key is positive infinity (null)
+ * @throws IllegalArgumentException if infiniteStartKey is true but start is not null, or infiniteStopKey is true but stop is not null
+ */
+ protected Range(Key start, boolean startKeyInclusive, boolean infiniteStartKey, Key stop, boolean stopKeyInclusive, boolean infiniteStopKey) {
+ if (infiniteStartKey && start != null)
+ throw new IllegalArgumentException();
+
+ if (infiniteStopKey && stop != null)
+ throw new IllegalArgumentException();
+
+ this.start = start;
+ this.stop = stop;
+ this.startKeyInclusive = startKeyInclusive;
+ this.stopKeyInclusive = stopKeyInclusive;
+ this.infiniteStartKey = infiniteStartKey;
+ this.infiniteStopKey = infiniteStopKey;
+ }
+
+ public Range(TRange trange) {
+ this(trange.start == null ? null : new Key(trange.start), trange.startKeyInclusive, trange.infiniteStartKey,
+ trange.stop == null ? null : new Key(trange.stop), trange.stopKeyInclusive, trange.infiniteStopKey);
+ if (!infiniteStartKey && !infiniteStopKey && beforeStartKey(stop)) {
+ throw new IllegalArgumentException("Start key must be less than end key in range (" + start + ", " + stop + ")");
+ }
+ }
+
+ /**
+ * @return the first key in the range, null if the key is infinite
+ */
+ public Key getStartKey() {
+ if (infiniteStartKey) {
+ return null;
+ }
+ return start;
+ }
+
+ /**
+ *
- * @param key
+ * @return true if the given key is before the range, otherwise false
+ */
+ public boolean beforeStartKey(Key key) {
+ if (infiniteStartKey) {
+ return false;
+ }
+
+ if (startKeyInclusive)
+ return key.compareTo(start) < 0;
+ return key.compareTo(start) <= 0;
+ }
+
+ /**
+ * @return the last key in the range, null if the end key is infinite
+ */
+
+ public Key getEndKey() {
+ if (infiniteStopKey) {
+ return null;
+ }
+ return stop;
+ }
+
+ /**
- * @param key
+ * @return true if the given key is after the range, otherwise false
+ */
+
+ public boolean afterEndKey(Key key) {
+ if (infiniteStopKey)
+ return false;
+
+ if (stopKeyInclusive)
+ return stop.compareTo(key) < 0;
+ return stop.compareTo(key) <= 0;
+ }
+
+ @Override
+ public int hashCode() {
+ int startHash = infiniteStartKey ? 0 : start.hashCode() + (startKeyInclusive ? 1 : 0);
+ int stopHash = infiniteStopKey ? 0 : stop.hashCode() + (stopKeyInclusive ? 1 : 0);
+
+ return startHash + stopHash;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof Range)
+ return equals((Range) o);
+ return false;
+ }
+
+ public boolean equals(Range otherRange) {
+
+ return compareTo(otherRange) == 0;
+ }
+
+ /**
+ * Compares this range to another range. Compares in the order start key, inclusiveness of start key, end key, inclusiveness of end key. Infinite keys sort
+ * first, and non-infinite keys are compared with {@link Key#compareTo(Key)}. Inclusive sorts before non-inclusive.
+ */
++ @Override
+ public int compareTo(Range o) {
+ int comp;
+
+ if (infiniteStartKey)
+ if (o.infiniteStartKey)
+ comp = 0;
+ else
+ comp = -1;
+ else if (o.infiniteStartKey)
+ comp = 1;
+ else {
+ comp = start.compareTo(o.start);
+ if (comp == 0)
+ if (startKeyInclusive && !o.startKeyInclusive)
+ comp = -1;
+ else if (!startKeyInclusive && o.startKeyInclusive)
+ comp = 1;
+
+ }
+
+ if (comp == 0)
+ if (infiniteStopKey)
+ if (o.infiniteStopKey)
+ comp = 0;
+ else
+ comp = 1;
+ else if (o.infiniteStopKey)
+ comp = -1;
+ else {
+ comp = stop.compareTo(o.stop);
+ if (comp == 0)
+ if (stopKeyInclusive && !o.stopKeyInclusive)
+ comp = 1;
+ else if (!stopKeyInclusive && o.stopKeyInclusive)
+ comp = -1;
+ }
+
+ return comp;
+ }
+
+ /**
+ *
- * @param key
+ * @return true if the given key falls within the range
+ */
+ public boolean contains(Key key) {
+ return !beforeStartKey(key) && !afterEndKey(key);
+ }
+
+ /**
+ * Takes a collection on range and merges overlapping and adjacent ranges. For example given the following input
+ *
+ * <pre>
+ * [a,c], (c, d], (g,m), (j,t]
+ * </pre>
+ *
+ * the following ranges would be returned
+ *
+ * <pre>
+ * [a,d], (g,t]
+ * </pre>
+ *
- * @param ranges
+ * @return list of merged ranges
+ */
+
+ public static List<Range> mergeOverlapping(Collection<Range> ranges) {
+ if (ranges.size() == 0)
+ return Collections.emptyList();
+
+ List<Range> ral = new ArrayList<Range>(ranges);
+ Collections.sort(ral);
+
+ ArrayList<Range> ret = new ArrayList<Range>(ranges.size());
+
+ Range currentRange = ral.get(0);
+ boolean currentStartKeyInclusive = ral.get(0).startKeyInclusive;
+
+ for (int i = 1; i < ral.size(); i++) {
+ // because of inclusive switch, equal keys may not be seen
+
+ if (currentRange.infiniteStopKey) {
+ // this range has the minimal start key and
+ // an infinite end key so it will contain all
+ // other ranges
+ break;
+ }
+
+ Range range = ral.get(i);
+
+ boolean startKeysEqual;
+ if (range.infiniteStartKey) {
+ // previous start key must be infinite because it is sorted
+ assert (currentRange.infiniteStartKey);
+ startKeysEqual = true;
+ } else if (currentRange.infiniteStartKey) {
+ startKeysEqual = false;
+ } else if (currentRange.start.equals(range.start)) {
+ startKeysEqual = true;
+ } else {
+ startKeysEqual = false;
+ }
+
+ if (startKeysEqual || currentRange.contains(range.start)
+ || (!currentRange.stopKeyInclusive && range.startKeyInclusive && range.start.equals(currentRange.stop))) {
+ int cmp;
+
+ if (range.infiniteStopKey || (cmp = range.stop.compareTo(currentRange.stop)) > 0 || (cmp == 0 && range.stopKeyInclusive)) {
+ currentRange = new Range(currentRange.getStartKey(), currentStartKeyInclusive, range.getEndKey(), range.stopKeyInclusive);
+ }/* else currentRange contains ral.get(i) */
+ } else {
+ ret.add(currentRange);
+ currentRange = range;
+ currentStartKeyInclusive = range.startKeyInclusive;
+ }
+ }
+
+ ret.add(currentRange);
+
+ return ret;
+ }
+
+ /**
+ * Creates a range which represents the intersection of this range and the passed in range. The following example will print true.
+ *
+ * <pre>
+ * Range range1 = new Range("a", "f");
+ * Range range2 = new Range("c", "n");
+ * Range range3 = range1.clip(range2);
+ * System.out.println(range3.equals(new Range("c", "f")));
+ * </pre>
+ *
- * @param range
+ * @return the intersection
+ * @throws IllegalArgumentException
+ * if ranges does not overlap
+ */
+
+ public Range clip(Range range) {
+ return clip(range, false);
+ }
+
+ /**
+ * Same as other clip function except if gives the option to return null of the ranges do not overlap instead of throwing an exception.
+ *
+ * @see Range#clip(Range)
- * @param range
+ * @param returnNullIfDisjoint
+ * If the ranges do not overlap and true is passed, then null is returned otherwise an exception is thrown.
+ * @return the intersection
+ */
+
+ public Range clip(Range range, boolean returnNullIfDisjoint) {
+
+ Key sk = range.getStartKey();
+ boolean ski = range.isStartKeyInclusive();
+
+ Key ek = range.getEndKey();
+ boolean eki = range.isEndKeyInclusive();
+
+ if (range.getStartKey() == null) {
+ if (getStartKey() != null) {
+ sk = getStartKey();
+ ski = isStartKeyInclusive();
+ }
+ } else if (afterEndKey(range.getStartKey())
+ || (getEndKey() != null && range.getStartKey().equals(getEndKey()) && !(range.isStartKeyInclusive() && isEndKeyInclusive()))) {
+ if (returnNullIfDisjoint)
+ return null;
+ throw new IllegalArgumentException("Range " + range + " does not overlap " + this);
+ } else if (beforeStartKey(range.getStartKey())) {
+ sk = getStartKey();
+ ski = isStartKeyInclusive();
+ }
+
+ if (range.getEndKey() == null) {
+ if (getEndKey() != null) {
+ ek = getEndKey();
+ eki = isEndKeyInclusive();
+ }
+ } else if (beforeStartKey(range.getEndKey())
+ || (getStartKey() != null && range.getEndKey().equals(getStartKey()) && !(range.isEndKeyInclusive() && isStartKeyInclusive()))) {
+ if (returnNullIfDisjoint)
+ return null;
+ throw new IllegalArgumentException("Range " + range + " does not overlap " + this);
+ } else if (afterEndKey(range.getEndKey())) {
+ ek = getEndKey();
+ eki = isEndKeyInclusive();
+ }
+
+ return new Range(sk, ski, ek, eki);
+ }
+
+ /**
+ * Creates a new range that is bounded by the columns passed in. The stary key in the returned range will have a column >= to the minimum column. The end key
+ * in the returned range will have a column <= the max column.
+ *
- *
- * @param min
- * @param max
+ * @return a column bounded range
+ * @throws IllegalArgumentException
+ * if min > max
+ */
+
+ public Range bound(Column min, Column max) {
+
+ if (min.compareTo(max) > 0) {
+ throw new IllegalArgumentException("min column > max column " + min + " " + max);
+ }
+
+ Key sk = getStartKey();
+ boolean ski = isStartKeyInclusive();
+
+ if (sk != null) {
+
+ ByteSequence cf = sk.getColumnFamilyData();
+ ByteSequence cq = sk.getColumnQualifierData();
+
+ ByteSequence mincf = new ArrayByteSequence(min.columnFamily);
+ ByteSequence mincq;
+
+ if (min.columnQualifier != null)
+ mincq = new ArrayByteSequence(min.columnQualifier);
+ else
+ mincq = new ArrayByteSequence(new byte[0]);
+
+ int cmp = cf.compareTo(mincf);
+
+ if (cmp < 0 || (cmp == 0 && cq.compareTo(mincq) < 0)) {
+ ski = true;
+ sk = new Key(sk.getRowData().toArray(), mincf.toArray(), mincq.toArray(), new byte[0], Long.MAX_VALUE, true);
+ }
+ }
+
+ Key ek = getEndKey();
+ boolean eki = isEndKeyInclusive();
+
+ if (ek != null) {
+ ByteSequence row = ek.getRowData();
+ ByteSequence cf = ek.getColumnFamilyData();
+ ByteSequence cq = ek.getColumnQualifierData();
+ ByteSequence cv = ek.getColumnVisibilityData();
+
+ ByteSequence maxcf = new ArrayByteSequence(max.columnFamily);
+ ByteSequence maxcq = null;
+ if (max.columnQualifier != null)
+ maxcq = new ArrayByteSequence(max.columnQualifier);
+
+ boolean set = false;
+
+ int comp = cf.compareTo(maxcf);
+
+ if (comp > 0) {
+ set = true;
+ } else if (comp == 0 && maxcq != null && cq.compareTo(maxcq) > 0) {
+ set = true;
+ } else if (!eki && row.length() > 0 && row.byteAt(row.length() - 1) == 0 && cf.length() == 0 && cq.length() == 0 && cv.length() == 0
+ && ek.getTimestamp() == Long.MAX_VALUE) {
+ row = row.subSequence(0, row.length() - 1);
+ set = true;
+ }
+
+ if (set) {
+ eki = false;
+ if (maxcq == null)
+ ek = new Key(row.toArray(), maxcf.toArray(), new byte[0], new byte[0], 0, false).followingKey(PartialKey.ROW_COLFAM);
+ else
+ ek = new Key(row.toArray(), maxcf.toArray(), maxcq.toArray(), new byte[0], 0, false).followingKey(PartialKey.ROW_COLFAM_COLQUAL);
+ }
+ }
+
+ return new Range(sk, ski, ek, eki);
+ }
+
++ @Override
+ public String toString() {
+ return ((startKeyInclusive && start != null) ? "[" : "(") + (start == null ? "-inf" : start) + "," + (stop == null ? "+inf" : stop)
+ + ((stopKeyInclusive && stop != null) ? "]" : ")");
+ }
+
++ @Override
+ public void readFields(DataInput in) throws IOException {
+ infiniteStartKey = in.readBoolean();
+ infiniteStopKey = in.readBoolean();
+ if (!infiniteStartKey) {
+ start = new Key();
+ start.readFields(in);
+ } else {
+ start = null;
+ }
+
+ if (!infiniteStopKey) {
+ stop = new Key();
+ stop.readFields(in);
+ } else {
+ stop = null;
+ }
+
+ startKeyInclusive = in.readBoolean();
+ stopKeyInclusive = in.readBoolean();
+
+ if (!infiniteStartKey && !infiniteStopKey && beforeStartKey(stop)) {
+ throw new InvalidObjectException("Start key must be less than end key in range (" + start + ", " + stop + ")");
+ }
+ }
+
++ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeBoolean(infiniteStartKey);
+ out.writeBoolean(infiniteStopKey);
+ if (!infiniteStartKey)
+ start.write(out);
+ if (!infiniteStopKey)
+ stop.write(out);
+ out.writeBoolean(startKeyInclusive);
+ out.writeBoolean(stopKeyInclusive);
+ }
+
+ public boolean isStartKeyInclusive() {
+ return startKeyInclusive;
+ }
+
+ public boolean isEndKeyInclusive() {
+ return stopKeyInclusive;
+ }
+
+ public TRange toThrift() {
+ return new TRange(start == null ? null : start.toThrift(), stop == null ? null : stop.toThrift(), startKeyInclusive, stopKeyInclusive, infiniteStartKey,
+ infiniteStopKey);
+ }
+
+ public boolean isInfiniteStartKey() {
+ return infiniteStartKey;
+ }
+
+ public boolean isInfiniteStopKey() {
+ return infiniteStopKey;
+ }
+
+ /**
+ * Creates a range that covers an exact row Returns the same Range as new Range(row)
+ *
+ * @param row
+ * all keys in the range will have this row
+ */
+ public static Range exact(Text row) {
+ return new Range(row);
+ }
+
+ /**
+ * Creates a range that covers an exact row and column family
+ *
+ * @param row
+ * all keys in the range will have this row
+ *
+ * @param cf
+ * all keys in the range will have this column family
+ */
+ public static Range exact(Text row, Text cf) {
+ Key startKey = new Key(row, cf);
+ return new Range(startKey, true, startKey.followingKey(PartialKey.ROW_COLFAM), false);
+ }
+
+ /**
+ * Creates a range that covers an exact row, column family, and column qualifier
+ *
+ * @param row
+ * all keys in the range will have this row
+ *
+ * @param cf
+ * all keys in the range will have this column family
+ *
+ * @param cq
+ * all keys in the range will have this column qualifier
+ */
+ public static Range exact(Text row, Text cf, Text cq) {
+ Key startKey = new Key(row, cf, cq);
+ return new Range(startKey, true, startKey.followingKey(PartialKey.ROW_COLFAM_COLQUAL), false);
+ }
+
+ /**
+ * Creates a range that covers an exact row, column family, column qualifier, and visibility
+ *
+ * @param row
+ * all keys in the range will have this row
+ *
+ * @param cf
+ * all keys in the range will have this column family
+ *
+ * @param cq
+ * all keys in the range will have this column qualifier
+ *
+ * @param cv
+ * all keys in the range will have this column visibility
+ */
+ public static Range exact(Text row, Text cf, Text cq, Text cv) {
+ Key startKey = new Key(row, cf, cq, cv);
+ return new Range(startKey, true, startKey.followingKey(PartialKey.ROW_COLFAM_COLQUAL_COLVIS), false);
+ }
+
+ /**
+ * Creates a range that covers an exact row, column family, column qualifier, visibility, and timestamp
+ *
+ * @param row
+ * all keys in the range will have this row
+ *
+ * @param cf
+ * all keys in the range will have this column family
+ *
+ * @param cq
+ * all keys in the range will have this column qualifier
+ *
+ * @param cv
+ * all keys in the range will have this column visibility
+ *
+ * @param ts
+ * all keys in the range will have this timestamp
+ */
+ public static Range exact(Text row, Text cf, Text cq, Text cv, long ts) {
+ Key startKey = new Key(row, cf, cq, cv, ts);
+ return new Range(startKey, true, startKey.followingKey(PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME), false);
+ }
+
+ /**
+ * Returns a Text that sorts just after all Texts beginning with a prefix
- *
- * @param prefix
+ */
+ public static Text followingPrefix(Text prefix) {
+ byte[] prefixBytes = prefix.getBytes();
+
+ // find the last byte in the array that is not 0xff
+ int changeIndex = prefix.getLength() - 1;
+ while (changeIndex >= 0 && prefixBytes[changeIndex] == (byte) 0xff)
+ changeIndex--;
+ if (changeIndex < 0)
+ return null;
+
+ // copy prefix bytes into new array
+ byte[] newBytes = new byte[changeIndex + 1];
+ System.arraycopy(prefixBytes, 0, newBytes, 0, changeIndex + 1);
+
+ // increment the selected byte
+ newBytes[changeIndex]++;
+ return new Text(newBytes);
+ }
+
+ /**
+ * Returns a Range that covers all rows beginning with a prefix
+ *
+ * @param rowPrefix
+ * all keys in the range will have rows that begin with this prefix
+ */
+ public static Range prefix(Text rowPrefix) {
+ Text fp = followingPrefix(rowPrefix);
+ return new Range(new Key(rowPrefix), true, fp == null ? null : new Key(fp), false);
+ }
+
+ /**
+ * Returns a Range that covers all column families beginning with a prefix within a given row
+ *
+ * @param row
+ * all keys in the range will have this row
+ *
+ * @param cfPrefix
+ * all keys in the range will have column families that begin with this prefix
+ */
+ public static Range prefix(Text row, Text cfPrefix) {
+ Text fp = followingPrefix(cfPrefix);
+ return new Range(new Key(row, cfPrefix), true, fp == null ? new Key(row).followingKey(PartialKey.ROW) : new Key(row, fp), false);
+ }
+
+ /**
+ * Returns a Range that covers all column qualifiers beginning with a prefix within a given row and column family
+ *
+ * @param row
+ * all keys in the range will have this row
+ *
+ * @param cf
+ * all keys in the range will have this column family
+ *
+ * @param cqPrefix
+ * all keys in the range will have column qualifiers that begin with this prefix
+ */
+ public static Range prefix(Text row, Text cf, Text cqPrefix) {
+ Text fp = followingPrefix(cqPrefix);
+ return new Range(new Key(row, cf, cqPrefix), true, fp == null ? new Key(row, cf).followingKey(PartialKey.ROW_COLFAM) : new Key(row, cf, fp), false);
+ }
+
+ /**
+ * Returns a Range that covers all column visibilities beginning with a prefix within a given row, column family, and column qualifier
+ *
+ * @param row
+ * all keys in the range will have this row
+ *
+ * @param cf
+ * all keys in the range will have this column family
+ *
+ * @param cq
+ * all keys in the range will have this column qualifier
+ *
+ * @param cvPrefix
+ * all keys in the range will have column visibilities that begin with this prefix
+ */
+ public static Range prefix(Text row, Text cf, Text cq, Text cvPrefix) {
+ Text fp = followingPrefix(cvPrefix);
+ return new Range(new Key(row, cf, cq, cvPrefix), true, fp == null ? new Key(row, cf, cq).followingKey(PartialKey.ROW_COLFAM_COLQUAL) : new Key(row, cf, cq,
+ fp), false);
+ }
+
+ /**
+ * Creates a range that covers an exact row
+ *
+ * @see Range#exact(Text)
+ */
+ public static Range exact(CharSequence row) {
+ return Range.exact(new Text(row.toString()));
+ }
+
+ /**
+ * Creates a range that covers an exact row and column family
+ *
+ * @see Range#exact(Text, Text)
+ */
+ public static Range exact(CharSequence row, CharSequence cf) {
+ return Range.exact(new Text(row.toString()), new Text(cf.toString()));
+ }
+
+ /**
+ * Creates a range that covers an exact row, column family, and column qualifier
+ *
+ * @see Range#exact(Text, Text, Text)
+ */
+ public static Range exact(CharSequence row, CharSequence cf, CharSequence cq) {
+ return Range.exact(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()));
+ }
+
+ /**
+ * Creates a range that covers an exact row, column family, column qualifier, and visibility
+ *
+ * @see Range#exact(Text, Text, Text, Text)
+ */
+ public static Range exact(CharSequence row, CharSequence cf, CharSequence cq, CharSequence cv) {
+ return Range.exact(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()), new Text(cv.toString()));
+ }
+
+ /**
+ * Creates a range that covers an exact row, column family, column qualifier, visibility, and timestamp
+ *
+ * @see Range#exact(Text, Text, Text, Text, long)
+ */
+ public static Range exact(CharSequence row, CharSequence cf, CharSequence cq, CharSequence cv, long ts) {
+ return Range.exact(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()), new Text(cv.toString()), ts);
+ }
+
+ /**
+ * Returns a Range that covers all rows beginning with a prefix
+ *
+ * @see Range#prefix(Text)
+ */
+ public static Range prefix(CharSequence rowPrefix) {
+ return Range.prefix(new Text(rowPrefix.toString()));
+ }
+
+ /**
+ * Returns a Range that covers all column families beginning with a prefix within a given row
+ *
+ * @see Range#prefix(Text, Text)
+ */
+ public static Range prefix(CharSequence row, CharSequence cfPrefix) {
+ return Range.prefix(new Text(row.toString()), new Text(cfPrefix.toString()));
+ }
+
+ /**
+ * Returns a Range that covers all column qualifiers beginning with a prefix within a given row and column family
+ *
+ * @see Range#prefix(Text, Text, Text)
+ */
+ public static Range prefix(CharSequence row, CharSequence cf, CharSequence cqPrefix) {
+ return Range.prefix(new Text(row.toString()), new Text(cf.toString()), new Text(cqPrefix.toString()));
+ }
+
+ /**
+ * Returns a Range that covers all column visibilities beginning with a prefix within a given row, column family, and column qualifier
+ *
+ * @see Range#prefix(Text, Text, Text, Text)
+ */
+ public static Range prefix(CharSequence row, CharSequence cf, CharSequence cq, CharSequence cvPrefix) {
+ return Range.prefix(new Text(row.toString()), new Text(cf.toString()), new Text(cq.toString()), new Text(cvPrefix.toString()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java
index 2b3cdf5,0000000..0ac5308
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java
@@@ -1,181 -1,0 +1,176 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.file.rfile;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.blockfile.ABlockReader;
+import org.apache.accumulo.core.file.rfile.MultiLevelIndex.IndexEntry;
+
+/**
+ *
+ */
+public class BlockIndex {
+
+ public static BlockIndex getIndex(ABlockReader cacheBlock, IndexEntry indexEntry) throws IOException {
+
+ BlockIndex blockIndex = cacheBlock.getIndex(BlockIndex.class);
+
+ int accessCount = blockIndex.accessCount.incrementAndGet();
+
+ // 1 is a power of two, but do not care about it
+ if (accessCount >= 2 && isPowerOfTwo(accessCount)) {
+ blockIndex.buildIndex(accessCount, cacheBlock, indexEntry);
+ }
+
+ if (blockIndex.blockIndex != null)
+ return blockIndex;
+
+ return null;
+ }
+
+ private static boolean isPowerOfTwo(int x) {
+ return ((x > 0) && (x & (x - 1)) == 0);
+ }
+
+ private AtomicInteger accessCount = new AtomicInteger(0);
+ private volatile BlockIndexEntry[] blockIndex = null;
+
+ public static class BlockIndexEntry implements Comparable<BlockIndexEntry> {
+
+ private Key prevKey;
+ private int entriesLeft;
+ private int pos;
+
+ public BlockIndexEntry(int pos, int entriesLeft, Key prevKey) {
+ this.pos = pos;
+ this.entriesLeft = entriesLeft;
+ this.prevKey = prevKey;
+ }
+
- /**
- * @param key
- */
+ public BlockIndexEntry(Key key) {
+ this.prevKey = key;
+ }
-
-
+
+ public int getEntriesLeft() {
+ return entriesLeft;
+ }
+
+ @Override
+ public int compareTo(BlockIndexEntry o) {
+ return prevKey.compareTo(o.prevKey);
+ }
+
+ @Override
+ public String toString() {
+ return prevKey + " " + entriesLeft + " " + pos;
+ }
+
+ public Key getPrevKey() {
+ return prevKey;
+ }
+ }
+
+ public BlockIndexEntry seekBlock(Key startKey, ABlockReader cacheBlock) {
+
+ // get a local ref to the index, another thread could change it
+ BlockIndexEntry[] blockIndex = this.blockIndex;
+
+ int pos = Arrays.binarySearch(blockIndex, new BlockIndexEntry(startKey));
+
+ int index;
+
+ if (pos < 0) {
+ if (pos == -1)
+ return null; // less than the first key in index, did not index the first key in block so just return null... code calling this will scan from beginning
+ // of block
+ index = (pos * -1) - 2;
+ } else {
+ // found exact key in index
+ index = pos;
+ while (index > 0) {
+ if (blockIndex[index].getPrevKey().equals(startKey))
+ index--;
+ else
+ break;
+ }
+ }
+
+ // handle case where multiple keys in block are exactly the same, want to find the earliest key in the index
+ while (index - 1 > 0) {
+ if (blockIndex[index].getPrevKey().equals(blockIndex[index - 1].getPrevKey()))
+ index--;
+ else
+ break;
+
+ }
+
+ if (index == 0 && blockIndex[index].getPrevKey().equals(startKey))
+ return null;
+
+ BlockIndexEntry bie = blockIndex[index];
+ cacheBlock.seek(bie.pos);
+ return bie;
+ }
+
+ private synchronized void buildIndex(int indexEntries, ABlockReader cacheBlock, IndexEntry indexEntry) throws IOException {
+ cacheBlock.seek(0);
+
+ RelativeKey rk = new RelativeKey();
+ Value val = new Value();
+
+ int interval = indexEntry.getNumEntries() / indexEntries;
+
+ if (interval <= 32)
+ return;
+
+ // multiple threads could try to create the index with different sizes, do not replace a large index with a smaller one
+ if (this.blockIndex != null && this.blockIndex.length > indexEntries - 1)
+ return;
+
+ int count = 0;
+
+ ArrayList<BlockIndexEntry> index = new ArrayList<BlockIndexEntry>(indexEntries - 1);
+
+ while (count < (indexEntry.getNumEntries() - interval + 1)) {
+
+ Key myPrevKey = rk.getKey();
+ int pos = cacheBlock.getPosition();
+ rk.readFields(cacheBlock);
+ val.readFields(cacheBlock);
+
+ if (count > 0 && count % interval == 0) {
+ index.add(new BlockIndexEntry(pos, indexEntry.getNumEntries() - count, myPrevKey));
+ }
+
+ count++;
+ }
+
+ this.blockIndex = index.toArray(new BlockIndexEntry[index.size()]);
+
+ cacheBlock.seek(0);
+ }
+
+ BlockIndexEntry[] getIndexEntries() {
+ return blockIndex;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index 7277c65,0000000..7d15851
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@@ -1,971 -1,0 +1,965 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.accumulo.core.file.rfile.bcfile;
+
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile;
+import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile.BlockRead;
+import org.apache.accumulo.core.file.rfile.bcfile.CompareUtils.Scalar;
+import org.apache.accumulo.core.file.rfile.bcfile.CompareUtils.ScalarComparator;
+import org.apache.accumulo.core.file.rfile.bcfile.CompareUtils.ScalarLong;
+import org.apache.accumulo.core.file.rfile.bcfile.Compression.Algorithm;
+import org.apache.accumulo.core.file.rfile.bcfile.Utils.Version;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.compress.Compressor;
+import org.apache.hadoop.io.compress.Decompressor;
+
+/**
+ * Block Compressed file, the underlying physical storage layer for TFile. BCFile provides the basic block level compression for the data block and meta blocks.
+ * It is separated from TFile as it may be used for other block-compressed file implementation.
+ */
+public final class BCFile {
+ // the current version of BCFile impl, increment them (major or minor) made
+ // enough changes
+ static final Version API_VERSION = new Version((short) 1, (short) 0);
+ static final Log LOG = LogFactory.getLog(BCFile.class);
+
+ /**
+ * Prevent the instantiation of BCFile objects.
+ */
+ private BCFile() {
+ // nothing
+ }
+
+ /**
+ * BCFile writer, the entry point for creating a new BCFile.
+ */
+ static public class Writer implements Closeable {
+ private final FSDataOutputStream out;
+ private final Configuration conf;
+ // the single meta block containing index of compressed data blocks
+ final DataIndex dataIndex;
+ // index for meta blocks
+ final MetaIndex metaIndex;
+ boolean blkInProgress = false;
+ private boolean metaBlkSeen = false;
+ private boolean closed = false;
+ long errorCount = 0;
+ // reusable buffers.
+ private BytesWritable fsOutputBuffer;
+
+ /**
+ * Call-back interface to register a block after a block is closed.
+ */
+ private static interface BlockRegister {
+ /**
+ * Register a block that is fully closed.
+ *
+ * @param raw
+ * The size of block in terms of uncompressed bytes.
+ * @param offsetStart
+ * The start offset of the block.
+ * @param offsetEnd
+ * One byte after the end of the block. Compressed block size is offsetEnd - offsetStart.
+ */
+ public void register(long raw, long offsetStart, long offsetEnd);
+ }
+
+ /**
+ * Intermediate class that maintain the state of a Writable Compression Block.
+ */
+ private static final class WBlockState {
+ private final Algorithm compressAlgo;
+ private Compressor compressor; // !null only if using native
+ // Hadoop compression
+ private final FSDataOutputStream fsOut;
+ private final long posStart;
+ private final SimpleBufferedOutputStream fsBufferedOutput;
+ private OutputStream out;
+
+ /**
+ * @param compressionAlgo
+ * The compression algorithm to be used to for compression.
- * @throws IOException
+ */
+ public WBlockState(Algorithm compressionAlgo, FSDataOutputStream fsOut, BytesWritable fsOutputBuffer, Configuration conf) throws IOException {
+ this.compressAlgo = compressionAlgo;
+ this.fsOut = fsOut;
+ this.posStart = fsOut.getPos();
+
+ fsOutputBuffer.setCapacity(TFile.getFSOutputBufferSize(conf));
+
+ this.fsBufferedOutput = new SimpleBufferedOutputStream(this.fsOut, fsOutputBuffer.getBytes());
+ this.compressor = compressAlgo.getCompressor();
+
+ try {
+ this.out = compressionAlgo.createCompressionStream(fsBufferedOutput, compressor, 0);
+ } catch (IOException e) {
+ compressAlgo.returnCompressor(compressor);
+ throw e;
+ }
+ }
+
+ /**
+ * Get the output stream for BlockAppender's consumption.
+ *
+ * @return the output stream suitable for writing block data.
+ */
+ OutputStream getOutputStream() {
+ return out;
+ }
+
+ /**
+ * Get the current position in file.
+ *
+ * @return The current byte offset in underlying file.
+ * @throws IOException
+ */
+ long getCurrentPos() throws IOException {
+ return fsOut.getPos() + fsBufferedOutput.size();
+ }
+
+ long getStartPos() {
+ return posStart;
+ }
+
+ /**
+ * Current size of compressed data.
+ *
+ * @throws IOException
+ */
+ long getCompressedSize() throws IOException {
+ long ret = getCurrentPos() - posStart;
+ return ret;
+ }
+
+ /**
+ * Finishing up the current block.
+ */
+ public void finish() throws IOException {
+ try {
+ if (out != null) {
+ out.flush();
+ out = null;
+ }
+ } finally {
+ compressAlgo.returnCompressor(compressor);
+ compressor = null;
+ }
+ }
+ }
+
+ /**
+ * Access point to stuff data into a block.
+ *
+ */
+ public class BlockAppender extends DataOutputStream {
+ private final BlockRegister blockRegister;
+ private final WBlockState wBlkState;
+ private boolean closed = false;
+
+ /**
+ * Constructor
+ *
+ * @param register
+ * the block register, which is called when the block is closed.
+ * @param wbs
+ * The writable compression block state.
+ */
+ BlockAppender(BlockRegister register, WBlockState wbs) {
+ super(wbs.getOutputStream());
+ this.blockRegister = register;
+ this.wBlkState = wbs;
+ }
+
+ /**
+ * Get the raw size of the block.
+ *
+ * @return the number of uncompressed bytes written through the BlockAppender so far.
- * @throws IOException
+ */
+ public long getRawSize() throws IOException {
+ /**
+ * Expecting the size() of a block not exceeding 4GB. Assuming the size() will wrap to negative integer if it exceeds 2GB.
+ */
+ return size() & 0x00000000ffffffffL;
+ }
+
+ /**
+ * Get the compressed size of the block in progress.
+ *
+ * @return the number of compressed bytes written to the underlying FS file. The size may be smaller than actual need to compress the all data written due
+ * to internal buffering inside the compressor.
- * @throws IOException
+ */
+ public long getCompressedSize() throws IOException {
+ return wBlkState.getCompressedSize();
+ }
+
+ public long getStartPos() {
+ return wBlkState.getStartPos();
+ }
+
+ @Override
+ public void flush() {
+ // The down stream is a special kind of stream that finishes a
+ // compression block upon flush. So we disable flush() here.
+ }
+
+ /**
+ * Signaling the end of write to the block. The block register will be called for registering the finished block.
+ */
+ @Override
+ public void close() throws IOException {
+ if (closed == true) {
+ return;
+ }
+ try {
+ ++errorCount;
+ wBlkState.finish();
+ blockRegister.register(getRawSize(), wBlkState.getStartPos(), wBlkState.getCurrentPos());
+ --errorCount;
+ } finally {
+ closed = true;
+ blkInProgress = false;
+ }
+ }
+ }
+
+ /**
+ * Constructor
+ *
+ * @param fout
+ * FS output stream.
+ * @param compressionName
+ * Name of the compression algorithm, which will be used for all data blocks.
- * @throws IOException
+ * @see Compression#getSupportedAlgorithms
+ */
+ public Writer(FSDataOutputStream fout, String compressionName, Configuration conf, boolean trackDataBlocks) throws IOException {
+ if (fout.getPos() != 0) {
+ throw new IOException("Output file not at zero offset.");
+ }
+
+ this.out = fout;
+ this.conf = conf;
+ dataIndex = new DataIndex(compressionName, trackDataBlocks);
+ metaIndex = new MetaIndex();
+ fsOutputBuffer = new BytesWritable();
+ Magic.write(fout);
+ }
+
+ /**
+ * Close the BCFile Writer. Attempting to use the Writer after calling <code>close</code> is not allowed and may lead to undetermined results.
+ */
++ @Override
+ public void close() throws IOException {
+ if (closed == true) {
+ return;
+ }
+
+ try {
+ if (errorCount == 0) {
+ if (blkInProgress == true) {
+ throw new IllegalStateException("Close() called with active block appender.");
+ }
+
+ // add metaBCFileIndex to metaIndex as the last meta block
+ BlockAppender appender = prepareMetaBlock(DataIndex.BLOCK_NAME, getDefaultCompressionAlgorithm());
+ try {
+ dataIndex.write(appender);
+ } finally {
+ appender.close();
+ }
+
+ long offsetIndexMeta = out.getPos();
+ metaIndex.write(out);
+
+ // Meta Index and the trailing section are written out directly.
+ out.writeLong(offsetIndexMeta);
+
+ API_VERSION.write(out);
+ Magic.write(out);
+ out.flush();
+ }
+ } finally {
+ closed = true;
+ }
+ }
+
+ private Algorithm getDefaultCompressionAlgorithm() {
+ return dataIndex.getDefaultCompressionAlgorithm();
+ }
+
+ private BlockAppender prepareMetaBlock(String name, Algorithm compressAlgo) throws IOException, MetaBlockAlreadyExists {
+ if (blkInProgress == true) {
+ throw new IllegalStateException("Cannot create Meta Block until previous block is closed.");
+ }
+
+ if (metaIndex.getMetaByName(name) != null) {
+ throw new MetaBlockAlreadyExists("name=" + name);
+ }
+
+ MetaBlockRegister mbr = new MetaBlockRegister(name, compressAlgo);
+ WBlockState wbs = new WBlockState(compressAlgo, out, fsOutputBuffer, conf);
+ BlockAppender ba = new BlockAppender(mbr, wbs);
+ blkInProgress = true;
+ metaBlkSeen = true;
+ return ba;
+ }
+
+ /**
+ * Create a Meta Block and obtain an output stream for adding data into the block. There can only be one BlockAppender stream active at any time. Regular
+ * Blocks may not be created after the first Meta Blocks. The caller must call BlockAppender.close() to conclude the block creation.
+ *
+ * @param name
+ * The name of the Meta Block. The name must not conflict with existing Meta Blocks.
+ * @param compressionName
+ * The name of the compression algorithm to be used.
+ * @return The BlockAppender stream
- * @throws IOException
+ * @throws MetaBlockAlreadyExists
+ * If the meta block with the name already exists.
+ */
+ public BlockAppender prepareMetaBlock(String name, String compressionName) throws IOException, MetaBlockAlreadyExists {
+ return prepareMetaBlock(name, Compression.getCompressionAlgorithmByName(compressionName));
+ }
+
+ /**
+ * Create a Meta Block and obtain an output stream for adding data into the block. The Meta Block will be compressed with the same compression algorithm as
+ * data blocks. There can only be one BlockAppender stream active at any time. Regular Blocks may not be created after the first Meta Blocks. The caller
+ * must call BlockAppender.close() to conclude the block creation.
+ *
+ * @param name
+ * The name of the Meta Block. The name must not conflict with existing Meta Blocks.
+ * @return The BlockAppender stream
+ * @throws MetaBlockAlreadyExists
+ * If the meta block with the name already exists.
- * @throws IOException
+ */
+ public BlockAppender prepareMetaBlock(String name) throws IOException, MetaBlockAlreadyExists {
+ return prepareMetaBlock(name, getDefaultCompressionAlgorithm());
+ }
+
+ /**
+ * Create a Data Block and obtain an output stream for adding data into the block. There can only be one BlockAppender stream active at any time. Data
+ * Blocks may not be created after the first Meta Blocks. The caller must call BlockAppender.close() to conclude the block creation.
+ *
+ * @return The BlockAppender stream
- * @throws IOException
+ */
+ public BlockAppender prepareDataBlock() throws IOException {
+ if (blkInProgress == true) {
+ throw new IllegalStateException("Cannot create Data Block until previous block is closed.");
+ }
+
+ if (metaBlkSeen == true) {
+ throw new IllegalStateException("Cannot create Data Block after Meta Blocks.");
+ }
+
+ DataBlockRegister dbr = new DataBlockRegister();
+
+ WBlockState wbs = new WBlockState(getDefaultCompressionAlgorithm(), out, fsOutputBuffer, conf);
+ BlockAppender ba = new BlockAppender(dbr, wbs);
+ blkInProgress = true;
+ return ba;
+ }
+
+ /**
+ * Callback to make sure a meta block is added to the internal list when its stream is closed.
+ */
+ private class MetaBlockRegister implements BlockRegister {
+ private final String name;
+ private final Algorithm compressAlgo;
+
+ MetaBlockRegister(String name, Algorithm compressAlgo) {
+ this.name = name;
+ this.compressAlgo = compressAlgo;
+ }
+
++ @Override
+ public void register(long raw, long begin, long end) {
+ metaIndex.addEntry(new MetaIndexEntry(name, compressAlgo, new BlockRegion(begin, end - begin, raw)));
+ }
+ }
+
+ /**
+ * Callback to make sure a data block is added to the internal list when it's being closed.
+ *
+ */
+ private class DataBlockRegister implements BlockRegister {
+ DataBlockRegister() {
+ // do nothing
+ }
+
++ @Override
+ public void register(long raw, long begin, long end) {
+ dataIndex.addBlockRegion(new BlockRegion(begin, end - begin, raw));
+ }
+ }
+ }
+
+ /**
+ * BCFile Reader, interface to read the file's data and meta blocks.
+ */
+ static public class Reader implements Closeable {
+ private static final String META_NAME = "BCFile.metaindex";
+ private final FSDataInputStream in;
+ private final Configuration conf;
+ final DataIndex dataIndex;
+ // Index for meta blocks
+ final MetaIndex metaIndex;
+ final Version version;
+
+ /**
+ * Intermediate class that maintain the state of a Readable Compression Block.
+ */
+ static private final class RBlockState {
+ private final Algorithm compressAlgo;
+ private Decompressor decompressor;
+ private final BlockRegion region;
+ private final InputStream in;
+
+ public RBlockState(Algorithm compressionAlgo, FSDataInputStream fsin, BlockRegion region, Configuration conf) throws IOException {
+ this.compressAlgo = compressionAlgo;
+ this.region = region;
+ this.decompressor = compressionAlgo.getDecompressor();
+
+ try {
+ this.in = compressAlgo.createDecompressionStream(new BoundedRangeFileInputStream(fsin, this.region.getOffset(), this.region.getCompressedSize()),
+ decompressor, TFile.getFSInputBufferSize(conf));
+ } catch (IOException e) {
+ compressAlgo.returnDecompressor(decompressor);
+ throw e;
+ }
+ }
+
+ /**
+ * Get the output stream for BlockAppender's consumption.
+ *
+ * @return the output stream suitable for writing block data.
+ */
+ public InputStream getInputStream() {
+ return in;
+ }
+
+ public String getCompressionName() {
+ return compressAlgo.getName();
+ }
+
+ public BlockRegion getBlockRegion() {
+ return region;
+ }
+
+ public void finish() throws IOException {
+ try {
+ in.close();
+ } finally {
+ compressAlgo.returnDecompressor(decompressor);
+ decompressor = null;
+ }
+ }
+ }
+
+ /**
+ * Access point to read a block.
+ */
+ public static class BlockReader extends DataInputStream {
+ private final RBlockState rBlkState;
+ private boolean closed = false;
+
+ BlockReader(RBlockState rbs) {
+ super(rbs.getInputStream());
+ rBlkState = rbs;
+ }
+
+ /**
+ * Finishing reading the block. Release all resources.
+ */
+ @Override
+ public void close() throws IOException {
+ if (closed == true) {
+ return;
+ }
+ try {
+ // Do not set rBlkState to null. People may access stats after calling
+ // close().
+ rBlkState.finish();
+ } finally {
+ closed = true;
+ }
+ }
+
+ /**
+ * Get the name of the compression algorithm used to compress the block.
+ *
+ * @return name of the compression algorithm.
+ */
+ public String getCompressionName() {
+ return rBlkState.getCompressionName();
+ }
+
+ /**
+ * Get the uncompressed size of the block.
+ *
+ * @return uncompressed size of the block.
+ */
+ public long getRawSize() {
+ return rBlkState.getBlockRegion().getRawSize();
+ }
+
+ /**
+ * Get the compressed size of the block.
+ *
+ * @return compressed size of the block.
+ */
+ public long getCompressedSize() {
+ return rBlkState.getBlockRegion().getCompressedSize();
+ }
+
+ /**
+ * Get the starting position of the block in the file.
+ *
+ * @return the starting position of the block in the file.
+ */
+ public long getStartPos() {
+ return rBlkState.getBlockRegion().getOffset();
+ }
+ }
+
+ /**
+ * Constructor
+ *
+ * @param fin
+ * FS input stream.
+ * @param fileLength
+ * Length of the corresponding file
- * @throws IOException
+ */
+ public Reader(FSDataInputStream fin, long fileLength, Configuration conf) throws IOException {
+ this.in = fin;
+ this.conf = conf;
+
+ // move the cursor to the beginning of the tail, containing: offset to the
+ // meta block index, version and magic
+ fin.seek(fileLength - Magic.size() - Version.size() - Long.SIZE / Byte.SIZE);
+ long offsetIndexMeta = fin.readLong();
+ version = new Version(fin);
+ Magic.readAndVerify(fin);
+
+ if (!version.compatibleWith(BCFile.API_VERSION)) {
+ throw new RuntimeException("Incompatible BCFile fileBCFileVersion.");
+ }
+
+ // read meta index
+ fin.seek(offsetIndexMeta);
+ metaIndex = new MetaIndex(fin);
+
+ // read data:BCFile.index, the data block index
+ BlockReader blockR = getMetaBlock(DataIndex.BLOCK_NAME);
+ try {
+ dataIndex = new DataIndex(blockR);
+ } finally {
+ blockR.close();
+ }
+ }
+
+ public Reader(CachableBlockFile.Reader cache, FSDataInputStream fin, long fileLength, Configuration conf) throws IOException {
+ this.in = fin;
+ this.conf = conf;
+
+ BlockRead cachedMetaIndex = cache.getCachedMetaBlock(META_NAME);
+ BlockRead cachedDataIndex = cache.getCachedMetaBlock(DataIndex.BLOCK_NAME);
+
+ if (cachedMetaIndex == null || cachedDataIndex == null) {
+ // move the cursor to the beginning of the tail, containing: offset to the
+ // meta block index, version and magic
+ fin.seek(fileLength - Magic.size() - Version.size() - Long.SIZE / Byte.SIZE);
+ long offsetIndexMeta = fin.readLong();
+ version = new Version(fin);
+ Magic.readAndVerify(fin);
+
+ if (!version.compatibleWith(BCFile.API_VERSION)) {
+ throw new RuntimeException("Incompatible BCFile fileBCFileVersion.");
+ }
+
+ // read meta index
+ fin.seek(offsetIndexMeta);
+ metaIndex = new MetaIndex(fin);
+ if (cachedMetaIndex == null) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ metaIndex.write(dos);
+ dos.close();
+ cache.cacheMetaBlock(META_NAME, baos.toByteArray());
+ }
+
+ // read data:BCFile.index, the data block index
+ if (cachedDataIndex == null) {
+ BlockReader blockR = getMetaBlock(DataIndex.BLOCK_NAME);
+ cachedDataIndex = cache.cacheMetaBlock(DataIndex.BLOCK_NAME, blockR);
+ }
+
+ dataIndex = new DataIndex(cachedDataIndex);
+ cachedDataIndex.close();
+
+ } else {
+ // Logger.getLogger(Reader.class).debug("Read bcfile !METADATA from cache");
+ version = null;
+ metaIndex = new MetaIndex(cachedMetaIndex);
+ dataIndex = new DataIndex(cachedDataIndex);
+ }
+ }
+
+ /**
+ * Get the name of the default compression algorithm.
+ *
+ * @return the name of the default compression algorithm.
+ */
+ public String getDefaultCompressionName() {
+ return dataIndex.getDefaultCompressionAlgorithm().getName();
+ }
+
+ /**
+ * Get version of BCFile file being read.
+ *
+ * @return version of BCFile file being read.
+ */
+ public Version getBCFileVersion() {
+ return version;
+ }
+
+ /**
+ * Get version of BCFile API.
+ *
+ * @return version of BCFile API.
+ */
+ public Version getAPIVersion() {
+ return API_VERSION;
+ }
+
+ /**
+ * Finishing reading the BCFile. Release all resources.
+ */
++ @Override
+ public void close() {
+ // nothing to be done now
+ }
+
+ /**
+ * Get the number of data blocks.
+ *
+ * @return the number of data blocks.
+ */
+ public int getBlockCount() {
+ return dataIndex.getBlockRegionList().size();
+ }
+
+ /**
+ * Stream access to a Meta Block.
+ *
+ * @param name
+ * meta block name
+ * @return BlockReader input stream for reading the meta block.
- * @throws IOException
+ * @throws MetaBlockDoesNotExist
+ * The Meta Block with the given name does not exist.
+ */
+ public BlockReader getMetaBlock(String name) throws IOException, MetaBlockDoesNotExist {
+ MetaIndexEntry imeBCIndex = metaIndex.getMetaByName(name);
+ if (imeBCIndex == null) {
+ throw new MetaBlockDoesNotExist("name=" + name);
+ }
+
+ BlockRegion region = imeBCIndex.getRegion();
+ return createReader(imeBCIndex.getCompressionAlgorithm(), region);
+ }
+
+ /**
+ * Stream access to a Data Block.
+ *
+ * @param blockIndex
+ * 0-based data block index.
+ * @return BlockReader input stream for reading the data block.
- * @throws IOException
+ */
+ public BlockReader getDataBlock(int blockIndex) throws IOException {
+ if (blockIndex < 0 || blockIndex >= getBlockCount()) {
+ throw new IndexOutOfBoundsException(String.format("blockIndex=%d, numBlocks=%d", blockIndex, getBlockCount()));
+ }
+
+ BlockRegion region = dataIndex.getBlockRegionList().get(blockIndex);
+ return createReader(dataIndex.getDefaultCompressionAlgorithm(), region);
+ }
+
+ public BlockReader getDataBlock(long offset, long compressedSize, long rawSize) throws IOException {
+ BlockRegion region = new BlockRegion(offset, compressedSize, rawSize);
+ return createReader(dataIndex.getDefaultCompressionAlgorithm(), region);
+ }
+
+ private BlockReader createReader(Algorithm compressAlgo, BlockRegion region) throws IOException {
+ RBlockState rbs = new RBlockState(compressAlgo, in, region, conf);
+ return new BlockReader(rbs);
+ }
+
+ /**
+ * Find the smallest Block index whose starting offset is greater than or equal to the specified offset.
+ *
+ * @param offset
+ * User-specific offset.
+ * @return the index to the data Block if such block exists; or -1 otherwise.
+ */
+ public int getBlockIndexNear(long offset) {
+ ArrayList<BlockRegion> list = dataIndex.getBlockRegionList();
+ int idx = Utils.lowerBound(list, new ScalarLong(offset), new ScalarComparator());
+
+ if (idx == list.size()) {
+ return -1;
+ }
+
+ return idx;
+ }
+ }
+
+ /**
+ * Index for all Meta blocks.
+ */
+ static class MetaIndex {
+ // use a tree map, for getting a meta block entry by name
+ final Map<String,MetaIndexEntry> index;
+
+ // for write
+ public MetaIndex() {
+ index = new TreeMap<String,MetaIndexEntry>();
+ }
+
+ // for read, construct the map from the file
+ public MetaIndex(DataInput in) throws IOException {
+ int count = Utils.readVInt(in);
+ index = new TreeMap<String,MetaIndexEntry>();
+
+ for (int nx = 0; nx < count; nx++) {
+ MetaIndexEntry indexEntry = new MetaIndexEntry(in);
+ index.put(indexEntry.getMetaName(), indexEntry);
+ }
+ }
+
+ public void addEntry(MetaIndexEntry indexEntry) {
+ index.put(indexEntry.getMetaName(), indexEntry);
+ }
+
+ public MetaIndexEntry getMetaByName(String name) {
+ return index.get(name);
+ }
+
+ public void write(DataOutput out) throws IOException {
+ Utils.writeVInt(out, index.size());
+
+ for (MetaIndexEntry indexEntry : index.values()) {
+ indexEntry.write(out);
+ }
+ }
+ }
+
+ /**
+ * An entry describes a meta block in the MetaIndex.
+ */
+ static final class MetaIndexEntry {
+ private final String metaName;
+ private final Algorithm compressionAlgorithm;
+ private final static String defaultPrefix = "data:";
+
+ private final BlockRegion region;
+
+ public MetaIndexEntry(DataInput in) throws IOException {
+ String fullMetaName = Utils.readString(in);
+ if (fullMetaName.startsWith(defaultPrefix)) {
+ metaName = fullMetaName.substring(defaultPrefix.length(), fullMetaName.length());
+ } else {
+ throw new IOException("Corrupted Meta region Index");
+ }
+
+ compressionAlgorithm = Compression.getCompressionAlgorithmByName(Utils.readString(in));
+ region = new BlockRegion(in);
+ }
+
+ public MetaIndexEntry(String metaName, Algorithm compressionAlgorithm, BlockRegion region) {
+ this.metaName = metaName;
+ this.compressionAlgorithm = compressionAlgorithm;
+ this.region = region;
+ }
+
+ public String getMetaName() {
+ return metaName;
+ }
+
+ public Algorithm getCompressionAlgorithm() {
+ return compressionAlgorithm;
+ }
+
+ public BlockRegion getRegion() {
+ return region;
+ }
+
+ public void write(DataOutput out) throws IOException {
+ Utils.writeString(out, defaultPrefix + metaName);
+ Utils.writeString(out, compressionAlgorithm.getName());
+
+ region.write(out);
+ }
+ }
+
+ /**
+ * Index of all compressed data blocks.
+ */
+ static class DataIndex {
+ final static String BLOCK_NAME = "BCFile.index";
+
+ private final Algorithm defaultCompressionAlgorithm;
+
+ // for data blocks, each entry specifies a block's offset, compressed size
+ // and raw size
+ private final ArrayList<BlockRegion> listRegions;
+
+ private boolean trackBlocks;
+
+ // for read, deserialized from a file
+ public DataIndex(DataInput in) throws IOException {
+ defaultCompressionAlgorithm = Compression.getCompressionAlgorithmByName(Utils.readString(in));
+
+ int n = Utils.readVInt(in);
+ listRegions = new ArrayList<BlockRegion>(n);
+
+ for (int i = 0; i < n; i++) {
+ BlockRegion region = new BlockRegion(in);
+ listRegions.add(region);
+ }
+ }
+
+ // for write
+ public DataIndex(String defaultCompressionAlgorithmName, boolean trackBlocks) {
+ this.trackBlocks = trackBlocks;
+ this.defaultCompressionAlgorithm = Compression.getCompressionAlgorithmByName(defaultCompressionAlgorithmName);
+ listRegions = new ArrayList<BlockRegion>();
+ }
+
+ public Algorithm getDefaultCompressionAlgorithm() {
+ return defaultCompressionAlgorithm;
+ }
+
+ public ArrayList<BlockRegion> getBlockRegionList() {
+ return listRegions;
+ }
+
+ public void addBlockRegion(BlockRegion region) {
+ if (trackBlocks)
+ listRegions.add(region);
+ }
+
+ public void write(DataOutput out) throws IOException {
+ Utils.writeString(out, defaultCompressionAlgorithm.getName());
+
+ Utils.writeVInt(out, listRegions.size());
+
+ for (BlockRegion region : listRegions) {
+ region.write(out);
+ }
+ }
+ }
+
+ /**
+ * Magic number uniquely identifying a BCFile in the header/footer.
+ */
+ static final class Magic {
+ private final static byte[] AB_MAGIC_BCFILE = {
+ // ... total of 16 bytes
+ (byte) 0xd1, (byte) 0x11, (byte) 0xd3, (byte) 0x68, (byte) 0x91, (byte) 0xb5, (byte) 0xd7, (byte) 0xb6, (byte) 0x39, (byte) 0xdf, (byte) 0x41,
+ (byte) 0x40, (byte) 0x92, (byte) 0xba, (byte) 0xe1, (byte) 0x50};
+
+ public static void readAndVerify(DataInput in) throws IOException {
+ byte[] abMagic = new byte[size()];
+ in.readFully(abMagic);
+
+ // check against AB_MAGIC_BCFILE, if not matching, throw an
+ // Exception
+ if (!Arrays.equals(abMagic, AB_MAGIC_BCFILE)) {
+ throw new IOException("Not a valid BCFile.");
+ }
+ }
+
+ public static void write(DataOutput out) throws IOException {
+ out.write(AB_MAGIC_BCFILE);
+ }
+
+ public static int size() {
+ return AB_MAGIC_BCFILE.length;
+ }
+ }
+
+ /**
+ * Block region.
+ */
+ static final class BlockRegion implements Scalar {
+ private final long offset;
+ private final long compressedSize;
+ private final long rawSize;
+
+ public BlockRegion(DataInput in) throws IOException {
+ offset = Utils.readVLong(in);
+ compressedSize = Utils.readVLong(in);
+ rawSize = Utils.readVLong(in);
+ }
+
+ public BlockRegion(long offset, long compressedSize, long rawSize) {
+ this.offset = offset;
+ this.compressedSize = compressedSize;
+ this.rawSize = rawSize;
+ }
+
+ public void write(DataOutput out) throws IOException {
+ Utils.writeVLong(out, offset);
+ Utils.writeVLong(out, compressedSize);
+ Utils.writeVLong(out, rawSize);
+ }
+
+ public long getOffset() {
+ return offset;
+ }
+
+ public long getCompressedSize() {
+ return compressedSize;
+ }
+
+ public long getRawSize() {
+ return rawSize;
+ }
+
+ @Override
+ public long magnitude() {
+ return offset;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/ByteArray.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/ByteArray.java
index 2b57638,0000000..d7734a2
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/ByteArray.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/ByteArray.java
@@@ -1,91 -1,0 +1,89 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.accumulo.core.file.rfile.bcfile;
+
+import org.apache.hadoop.io.BytesWritable;
+
+/**
+ * Adaptor class to wrap byte-array backed objects (including java byte array) as RawComparable objects.
+ */
+public final class ByteArray implements RawComparable {
+ private final byte[] buffer;
+ private final int offset;
+ private final int len;
+
+ /**
+ * Constructing a ByteArray from a {@link BytesWritable}.
- *
- * @param other
+ */
+ public ByteArray(BytesWritable other) {
+ this(other.getBytes(), 0, other.getLength());
+ }
+
+ /**
+ * Wrap a whole byte array as a RawComparable.
+ *
+ * @param buffer
+ * the byte array buffer.
+ */
+ public ByteArray(byte[] buffer) {
+ this(buffer, 0, buffer.length);
+ }
+
+ /**
+ * Wrap a partial byte array as a RawComparable.
+ *
+ * @param buffer
+ * the byte array buffer.
+ * @param offset
+ * the starting offset
+ * @param len
+ * the length of the consecutive bytes to be wrapped.
+ */
+ public ByteArray(byte[] buffer, int offset, int len) {
+ if ((offset | len | (buffer.length - offset - len)) < 0) {
+ throw new IndexOutOfBoundsException();
+ }
+ this.buffer = buffer;
+ this.offset = offset;
+ this.len = len;
+ }
+
+ /**
+ * @return the underlying buffer.
+ */
+ @Override
+ public byte[] buffer() {
+ return buffer;
+ }
+
+ /**
+ * @return the offset in the buffer.
+ */
+ @Override
+ public int offset() {
+ return offset;
+ }
+
+ /**
+ * @return the size of the byte array.
+ */
+ @Override
+ public int size() {
+ return len;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Chunk.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Chunk.java
index a075d87,0000000..345d406
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Chunk.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/Chunk.java
@@@ -1,418 -1,0 +1,416 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.accumulo.core.file.rfile.bcfile;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Several related classes to support chunk-encoded sub-streams on top of a regular stream.
+ */
+final class Chunk {
+
+ /**
+ * Prevent the instantiation of class.
+ */
+ private Chunk() {
+ // nothing
+ }
+
+ /**
+ * Decoding a chain of chunks encoded through ChunkEncoder or SingleChunkEncoder.
+ */
+ static public class ChunkDecoder extends InputStream {
+ private DataInputStream in = null;
+ private boolean lastChunk;
+ private int remain = 0;
+ private boolean closed;
+
+ public ChunkDecoder() {
+ lastChunk = true;
+ closed = true;
+ }
+
+ public void reset(DataInputStream downStream) {
+ // no need to wind forward the old input.
+ in = downStream;
+ lastChunk = false;
+ remain = 0;
+ closed = false;
+ }
+
+ /**
+ * Constructor
+ *
+ * @param in
+ * The source input stream which contains chunk-encoded data stream.
+ */
+ public ChunkDecoder(DataInputStream in) {
+ this.in = in;
+ lastChunk = false;
+ closed = false;
+ }
+
+ /**
+ * Have we reached the last chunk.
+ *
+ * @return true if we have reached the last chunk.
- * @throws java.io.IOException
+ */
+ public boolean isLastChunk() throws IOException {
+ checkEOF();
+ return lastChunk;
+ }
+
+ /**
+ * How many bytes remain in the current chunk?
+ *
+ * @return remaining bytes left in the current chunk.
- * @throws java.io.IOException
+ */
+ public int getRemain() throws IOException {
+ checkEOF();
+ return remain;
+ }
+
+ /**
+ * Reading the length of next chunk.
+ *
+ * @throws java.io.IOException
+ * when no more data is available.
+ */
+ private void readLength() throws IOException {
+ remain = Utils.readVInt(in);
+ if (remain >= 0) {
+ lastChunk = true;
+ } else {
+ remain = -remain;
+ }
+ }
+
+ /**
+ * Check whether we reach the end of the stream.
+ *
+ * @return false if the chunk encoded stream has more data to read (in which case available() will be greater than 0); true otherwise.
+ * @throws java.io.IOException
+ * on I/O errors.
+ */
+ private boolean checkEOF() throws IOException {
+ if (isClosed())
+ return true;
+ while (true) {
+ if (remain > 0)
+ return false;
+ if (lastChunk)
+ return true;
+ readLength();
+ }
+ }
+
+ @Override
+ /*
+ * This method never blocks the caller. Returning 0 does not mean we reach the end of the stream.
+ */
+ public int available() {
+ return remain;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (checkEOF())
+ return -1;
+ int ret = in.read();
+ if (ret < 0)
+ throw new IOException("Corrupted chunk encoding stream");
+ --remain;
+ return ret;
+ }
+
+ @Override
+ public int read(byte[] b) throws IOException {
+ return read(b, 0, b.length);
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ if ((off | len | (off + len) | (b.length - (off + len))) < 0) {
+ throw new IndexOutOfBoundsException();
+ }
+
+ if (!checkEOF()) {
+ int n = Math.min(remain, len);
+ int ret = in.read(b, off, n);
+ if (ret < 0)
+ throw new IOException("Corrupted chunk encoding stream");
+ remain -= ret;
+ return ret;
+ }
+ return -1;
+ }
+
+ @Override
+ public long skip(long n) throws IOException {
+ if (!checkEOF()) {
+ long ret = in.skip(Math.min(remain, n));
+ remain -= ret;
+ return ret;
+ }
+ return 0;
+ }
+
+ @Override
+ public boolean markSupported() {
+ return false;
+ }
+
+ public boolean isClosed() {
+ return closed;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed == false) {
+ try {
+ while (!checkEOF()) {
+ skip(Integer.MAX_VALUE);
+ }
+ } finally {
+ closed = true;
+ }
+ }
+ }
+ }
+
+ /**
+ * Chunk Encoder. Encoding the output data into a chain of chunks in the following sequences: -len1, byte[len1], -len2, byte[len2], ... len_n, byte[len_n].
+ * Where len1, len2, ..., len_n are the lengths of the data chunks. Non-terminal chunks have their lengths negated. Non-terminal chunks cannot have length 0.
+ * All lengths are in the range of 0 to Integer.MAX_VALUE and are encoded in Utils.VInt format.
+ */
+ static public class ChunkEncoder extends OutputStream {
+ /**
+ * The data output stream it connects to.
+ */
+ private DataOutputStream out;
+
+ /**
+ * The internal buffer that is only used when we do not know the advertised size.
+ */
+ private byte buf[];
+
+ /**
+ * The number of valid bytes in the buffer. This value is always in the range <tt>0</tt> through <tt>buf.length</tt>; elements <tt>buf[0]</tt> through
+ * <tt>buf[count-1]</tt> contain valid byte data.
+ */
+ private int count;
+
+ /**
+ * Constructor.
+ *
+ * @param out
+ * the underlying output stream.
+ * @param buf
+ * user-supplied buffer. The buffer would be used exclusively by the ChunkEncoder during its life cycle.
+ */
+ public ChunkEncoder(DataOutputStream out, byte[] buf) {
+ this.out = out;
+ this.buf = buf;
+ this.count = 0;
+ }
+
+ /**
+ * Write out a chunk.
+ *
+ * @param chunk
+ * The chunk buffer.
+ * @param offset
+ * Offset to chunk buffer for the beginning of chunk.
+ * @param len
+ * @param last
+ * Is this the last call to flushBuffer?
+ */
+ private void writeChunk(byte[] chunk, int offset, int len, boolean last) throws IOException {
+ if (last) { // always write out the length for the last chunk.
+ Utils.writeVInt(out, len);
+ if (len > 0) {
+ out.write(chunk, offset, len);
+ }
+ } else {
+ if (len > 0) {
+ Utils.writeVInt(out, -len);
+ out.write(chunk, offset, len);
+ }
+ }
+ }
+
+ /**
+ * Write out a chunk that is a concatenation of the internal buffer plus user supplied data. This will never be the last block.
+ *
+ * @param data
+ * User supplied data buffer.
+ * @param offset
+ * Offset to user data buffer.
+ * @param len
+ * User data buffer size.
+ */
+ private void writeBufData(byte[] data, int offset, int len) throws IOException {
+ if (count + len > 0) {
+ Utils.writeVInt(out, -(count + len));
+ out.write(buf, 0, count);
+ count = 0;
+ out.write(data, offset, len);
+ }
+ }
+
+ /**
+ * Flush the internal buffer.
+ *
+ * Is this the last call to flushBuffer?
+ *
+ * @throws java.io.IOException
+ */
+ private void flushBuffer() throws IOException {
+ if (count > 0) {
+ writeChunk(buf, 0, count, false);
+ count = 0;
+ }
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ if (count >= buf.length) {
+ flushBuffer();
+ }
+ buf[count++] = (byte) b;
+ }
+
+ @Override
+ public void write(byte b[]) throws IOException {
+ write(b, 0, b.length);
+ }
+
+ @Override
+ public void write(byte b[], int off, int len) throws IOException {
+ if ((len + count) >= buf.length) {
+ /*
+ * If the input data do not fit in buffer, flush the output buffer and then write the data directly. In this way buffered streams will cascade
+ * harmlessly.
+ */
+ writeBufData(b, off, len);
+ return;
+ }
+
+ System.arraycopy(b, off, buf, count, len);
+ count += len;
+ }
+
+ @Override
+ public void flush() throws IOException {
+ flushBuffer();
+ out.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (buf != null) {
+ try {
+ writeChunk(buf, 0, count, true);
+ } finally {
+ buf = null;
+ out = null;
+ }
+ }
+ }
+ }
+
+ /**
+ * Encode the whole stream as a single chunk. Expecting to know the size of the chunk up-front.
+ */
+ static public class SingleChunkEncoder extends OutputStream {
+ /**
+ * The data output stream it connects to.
+ */
+ private final DataOutputStream out;
+
+ /**
+ * The remaining bytes to be written.
+ */
+ private int remain;
+ private boolean closed = false;
+
+ /**
+ * Constructor.
+ *
+ * @param out
+ * the underlying output stream.
+ * @param size
+ * The total # of bytes to be written as a single chunk.
+ * @throws java.io.IOException
+ * if an I/O error occurs.
+ */
+ public SingleChunkEncoder(DataOutputStream out, int size) throws IOException {
+ this.out = out;
+ this.remain = size;
+ Utils.writeVInt(out, size);
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ if (remain > 0) {
+ out.write(b);
+ --remain;
+ } else {
+ throw new IOException("Writing more bytes than advertised size.");
+ }
+ }
+
+ @Override
+ public void write(byte b[]) throws IOException {
+ write(b, 0, b.length);
+ }
+
+ @Override
+ public void write(byte b[], int off, int len) throws IOException {
+ if (remain >= len) {
+ out.write(b, off, len);
+ remain -= len;
+ } else {
+ throw new IOException("Writing more bytes than advertised size.");
+ }
+ }
+
+ @Override
+ public void flush() throws IOException {
+ out.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed == true) {
+ return;
+ }
+
+ try {
+ if (remain > 0) {
+ throw new IOException("Writing less bytes than advertised size.");
+ }
+ } finally {
+ closed = true;
+ }
+ }
+ }
+}
[35/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
index 06b9a7c,0000000..70156b2
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
@@@ -1,223 -1,0 +1,222 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.continuous;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.ClientOnDefaultTable;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.server.util.reflection.CounterUtils;
+import org.apache.accumulo.test.continuous.ContinuousWalk.BadChecksumException;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.VLongWritable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.validators.PositiveInteger;
+
+/**
+ * A map reduce job that verifies a table created by continuous ingest. It verifies that all referenced nodes are defined.
+ */
+
+public class ContinuousVerify extends Configured implements Tool {
+ public static final VLongWritable DEF = new VLongWritable(-1);
+
+ public static class CMapper extends Mapper<Key,Value,LongWritable,VLongWritable> {
+
+ private LongWritable row = new LongWritable();
+ private LongWritable ref = new LongWritable();
+ private VLongWritable vrow = new VLongWritable();
+
+ private long corrupt = 0;
+
+ @Override
+ public void map(Key key, Value data, Context context) throws IOException, InterruptedException {
+ long r = Long.parseLong(key.getRow().toString(), 16);
+ if (r < 0)
+ throw new IllegalArgumentException();
+
+ try {
+ ContinuousWalk.validate(key, data);
+ } catch (BadChecksumException bce) {
+ CounterUtils.increment(context.getCounter(Counts.CORRUPT));
+ if (corrupt < 1000) {
+ System.out.println("ERROR Bad checksum : " + key);
+ } else if (corrupt == 1000) {
+ System.out.println("Too many bad checksums, not printing anymore!");
+ }
+ corrupt++;
+ return;
+ }
+
+ row.set(r);
+
+ context.write(row, DEF);
+ byte[] val = data.get();
+
+ int offset = ContinuousWalk.getPrevRowOffset(val);
+ if (offset > 0) {
+ ref.set(Long.parseLong(new String(val, offset, 16, Constants.UTF8), 16));
+ vrow.set(r);
+ context.write(ref, vrow);
+ }
+ }
+ }
+
+ public static enum Counts {
+ UNREFERENCED, UNDEFINED, REFERENCED, CORRUPT
+ }
+
+ public static class CReducer extends Reducer<LongWritable,VLongWritable,Text,Text> {
+ private ArrayList<Long> refs = new ArrayList<Long>();
+
+ @Override
+ public void reduce(LongWritable key, Iterable<VLongWritable> values, Context context) throws IOException, InterruptedException {
+
+ int defCount = 0;
+
+ refs.clear();
+ for (VLongWritable type : values) {
+ if (type.get() == -1) {
+ defCount++;
+ } else {
+ refs.add(type.get());
+ }
+ }
+
+ if (defCount == 0 && refs.size() > 0) {
+ StringBuilder sb = new StringBuilder();
+ String comma = "";
+ for (Long ref : refs) {
+ sb.append(comma);
+ comma = ",";
+ sb.append(new String(ContinuousIngest.genRow(ref), Constants.UTF8));
+ }
+
+ context.write(new Text(ContinuousIngest.genRow(key.get())), new Text(sb.toString()));
+ CounterUtils.increment(context.getCounter(Counts.UNDEFINED));
+
+ } else if (defCount > 0 && refs.size() == 0) {
+ CounterUtils.increment(context.getCounter(Counts.UNREFERENCED));
+ } else {
+ CounterUtils.increment(context.getCounter(Counts.REFERENCED));
+ }
+
+ }
+ }
+
+ static class Opts extends ClientOnDefaultTable {
+ @Parameter(names = "--output", description = "location in HDFS to store the results; must not exist", required = true)
+ String outputDir = "/tmp/continuousVerify";
+
+ @Parameter(names = "--maxMappers", description = "the maximum number of mappers to use", required = true, validateWith = PositiveInteger.class)
+ int maxMaps = 0;
+
+ @Parameter(names = "--reducers", description = "the number of reducers to use", required = true, validateWith = PositiveInteger.class)
+ int reducers = 0;
+
+ @Parameter(names = "--offline", description = "perform the verification directly on the files while the table is offline")
+ boolean scanOffline = false;
+
+ public Opts() {
+ super("ci");
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(this.getClass().getName(), args);
+
+ Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+ job.setJarByClass(this.getClass());
+
+ job.setInputFormatClass(AccumuloInputFormat.class);
+ opts.setAccumuloConfigs(job);
+
+ Set<Range> ranges = null;
+ String clone = opts.getTableName();
+ Connector conn = null;
+
+ if (opts.scanOffline) {
+ Random random = new Random();
+ clone = opts.getTableName() + "_" + String.format("%016x", (random.nextLong() & 0x7fffffffffffffffl));
+ conn = opts.getConnector();
+ conn.tableOperations().clone(opts.getTableName(), clone, true, new HashMap<String,String>(), new HashSet<String>());
+ ranges = conn.tableOperations().splitRangeByTablets(opts.getTableName(), new Range(), opts.maxMaps);
+ conn.tableOperations().offline(clone);
+ AccumuloInputFormat.setInputTableName(job, clone);
+ AccumuloInputFormat.setOfflineTableScan(job, true);
+ } else {
+ ranges = opts.getConnector().tableOperations().splitRangeByTablets(opts.getTableName(), new Range(), opts.maxMaps);
+ }
+
+ AccumuloInputFormat.setRanges(job, ranges);
+ AccumuloInputFormat.setAutoAdjustRanges(job, false);
+
+ job.setMapperClass(CMapper.class);
+ job.setMapOutputKeyClass(LongWritable.class);
+ job.setMapOutputValueClass(VLongWritable.class);
+
+ job.setReducerClass(CReducer.class);
+ job.setNumReduceTasks(opts.reducers);
+
+ job.setOutputFormatClass(TextOutputFormat.class);
+
+ job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", opts.scanOffline);
+
+ TextOutputFormat.setOutputPath(job, new Path(opts.outputDir));
+
+ job.waitForCompletion(true);
+
+ if (opts.scanOffline) {
+ conn.tableOperations().delete(clone);
+ }
+ opts.stopTracing();
+ return job.isSuccessful() ? 0 : 1;
+ }
+
+ /**
+ *
+ * @param args
+ * instanceName zookeepers username password table columns outputpath
- * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(CachedConfiguration.getInstance(), new ContinuousVerify(), args);
+ if (res != 0)
+ System.exit(res);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/functional/CacheTestClean.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/CacheTestClean.java
index c522914,0000000..f1dfcd2
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/CacheTestClean.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/CacheTestClean.java
@@@ -1,51 -1,0 +1,48 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import java.io.File;
+import java.util.Arrays;
+
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+
+public class CacheTestClean {
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ String rootDir = args[0];
+ File reportDir = new File(args[1]);
+
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ if (zoo.exists(rootDir)) {
+ zoo.recursiveDelete(rootDir, NodeMissingPolicy.FAIL);
+ }
+
+ if (!reportDir.exists()) {
+ reportDir.mkdir();
+ } else {
+ File[] files = reportDir.listFiles();
+ if (files.length != 0)
+ throw new Exception("dir " + reportDir + " is not empty: " + Arrays.asList(files));
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
index 2b775c5,0000000..06c6fdb
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
@@@ -1,217 -1,0 +1,213 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.server.util.reflection.CounterUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Runs the functional tests via map-reduce.
+ *
+ * First, be sure everything is compiled.
+ *
+ * Second, get a list of the tests you want to run:
+ *
+ * <pre>
+ * $ python test/system/auto/run.py -l > tests
+ * </pre>
+ *
+ * Put the list of tests into HDFS:
+ *
+ * <pre>
+ * $ hadoop fs -put tests /user/hadoop/tests
+ * </pre>
+ *
+ * Run the map-reduce job:
+ *
+ * <pre>
+ * $ ./bin/accumulo accumulo.test.functional.RunTests --tests /user/hadoop/tests --output /user/hadoop/results
+ * </pre>
+ *
+ * Note that you will need to have some configuration in conf/accumulo-site.xml (to locate zookeeper). The map-reduce jobs will not use your local accumulo
+ * instance.
+ *
+ */
+public class RunTests extends Configured implements Tool {
+
+ static final public String JOB_NAME = "Functional Test Runner";
+ private static final Logger log = Logger.getLogger(RunTests.class);
+
+ private Job job = null;
+
+ private static final int DEFAULT_TIMEOUT_FACTOR = 1;
+
+ static class Opts extends Help {
+ @Parameter(names="--tests", description="newline separated list of tests to run", required=true)
+ String testFile;
+ @Parameter(names="--output", description="destination for the results of tests in HDFS", required=true)
+ String outputPath;
+ @Parameter(names="--timeoutFactor", description="Optional scaling factor for timeout for both mapred.task.timeout and -f flag on run.py", required=false)
+ Integer intTimeoutFactor = DEFAULT_TIMEOUT_FACTOR;
+ }
+
+ static final String TIMEOUT_FACTOR = RunTests.class.getName() + ".timeoutFactor";
+
+ static public class TestMapper extends Mapper<LongWritable,Text,Text,Text> {
+
+ private static final String REDUCER_RESULT_START = "::::: ";
+ private static final int RRS_LEN = REDUCER_RESULT_START.length();
+ private Text result = new Text();
+ String mapperTimeoutFactor = null;
+
+ private static enum Outcome {
+ SUCCESS, FAILURE, ERROR, UNEXPECTED_SUCCESS, EXPECTED_FAILURE
+ }
+ private static final Map<Character, Outcome> OUTCOME_COUNTERS;
+ static {
+ OUTCOME_COUNTERS = new java.util.HashMap<Character, Outcome>();
+ OUTCOME_COUNTERS.put('S', Outcome.SUCCESS);
+ OUTCOME_COUNTERS.put('F', Outcome.FAILURE);
+ OUTCOME_COUNTERS.put('E', Outcome.ERROR);
+ OUTCOME_COUNTERS.put('T', Outcome.UNEXPECTED_SUCCESS);
+ OUTCOME_COUNTERS.put('G', Outcome.EXPECTED_FAILURE);
+ }
+
+ @Override
+ protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
+ List<String> cmd = Arrays.asList("/usr/bin/python", "test/system/auto/run.py", "-m", "-f", mapperTimeoutFactor, "-t", value.toString());
+ log.info("Running test " + cmd);
+ ProcessBuilder pb = new ProcessBuilder(cmd);
+ pb.directory(new File(context.getConfiguration().get("accumulo.home")));
+ pb.redirectErrorStream(true);
+ Process p = pb.start();
+ p.getOutputStream().close();
+ InputStream out = p.getInputStream();
+ InputStreamReader outr = new InputStreamReader(out, Constants.UTF8);
+ BufferedReader br = new BufferedReader(outr);
+ String line;
+ try {
+ while ((line = br.readLine()) != null) {
+ log.info("More: " + line);
+ if (line.startsWith(REDUCER_RESULT_START)) {
+ String resultLine = line.substring(RRS_LEN);
+ if (resultLine.length() > 0) {
+ Outcome outcome = OUTCOME_COUNTERS.get(resultLine.charAt(0));
+ if (outcome != null) {
+ CounterUtils.increment(context.getCounter(outcome));
+ }
+ }
+ String taskAttemptId = context.getTaskAttemptID().toString();
+ result.set(taskAttemptId + " " + resultLine);
+ context.write(value, result);
+ }
+ }
+ } catch (Exception ex) {
+ log.error(ex);
+ context.progress();
+ }
+
+ p.waitFor();
+ }
+
+ @Override
+ protected void setup(Mapper<LongWritable,Text,Text,Text>.Context context) throws IOException, InterruptedException {
+ mapperTimeoutFactor = Integer.toString(context.getConfiguration().getInt(TIMEOUT_FACTOR, DEFAULT_TIMEOUT_FACTOR));
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ job = new Job(getConf(), JOB_NAME);
+ job.setJarByClass(this.getClass());
+ Opts opts = new Opts();
+ opts.parseArgs(RunTests.class.getName(), args);
+
+ // this is like 1-2 tests per mapper
+ Configuration conf = job.getConfiguration();
+ conf.setInt("mapred.max.split.size", 40);
+ conf.set("accumulo.home", System.getenv("ACCUMULO_HOME"));
+
+ // Taking third argument as scaling factor to setting mapred.task.timeout
+ // and TIMEOUT_FACTOR
+ conf.setInt("mapred.task.timeout", opts.intTimeoutFactor * 8 * 60 * 1000);
+ conf.setInt(TIMEOUT_FACTOR, opts.intTimeoutFactor);
+ conf.setBoolean("mapred.map.tasks.speculative.execution", false);
+
+ // set input
+ job.setInputFormatClass(TextInputFormat.class);
+ TextInputFormat.setInputPaths(job, new Path(opts.testFile));
+
+ // set output
+ job.setOutputFormatClass(TextOutputFormat.class);
+ FileSystem fs = FileSystem.get(conf);
+ Path destination = new Path(opts.outputPath);
+ if (fs.exists(destination)) {
+ log.info("Deleting existing output directory " + opts.outputPath);
+ fs.delete(destination, true);
+ }
+ TextOutputFormat.setOutputPath(job, destination);
+
+ // configure default reducer: put the results into one file
+ job.setNumReduceTasks(1);
+
+ // set mapper
+ job.setMapperClass(TestMapper.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(Text.class);
+
+ // don't do anything with the results (yet) a summary would be nice
+ job.setNumReduceTasks(0);
+
+ // submit the job
+ log.info("Starting tests");
+ return 0;
+ }
+
- /**
- * @param args
- * @throws Exception
- */
+ public static void main(String[] args) throws Exception {
+ RunTests tests = new RunTests();
+ ToolRunner.run(new Configuration(), tests, args);
+ tests.job.waitForCompletion(true);
+ if (!tests.job.isSuccessful())
+ System.exit(1);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
index a9b072e,0000000..85cddbb
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
@@@ -1,246 -1,0 +1,243 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.performance.metadata;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.TreeSet;
+import java.util.UUID;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.util.Stat;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.hadoop.io.Text;
+
+/**
+ * This little program can be used to write a lot of entries to the !METADATA table and measure the performance of varying numbers of threads doing !METADATA
+ * lookups using the batch scanner.
+ *
+ *
+ */
+
+public class MetadataBatchScanTest {
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+
+ final Connector connector = new ZooKeeperInstance("acu14", "localhost")
+ .getConnector(SecurityConstants.SYSTEM_PRINCIPAL, SecurityConstants.getSystemToken());
+
+ TreeSet<Long> splits = new TreeSet<Long>();
+ Random r = new Random(42);
+
+ while (splits.size() < 99999) {
+ splits.add((r.nextLong() & 0x7fffffffffffffffl) % 1000000000000l);
+ }
+
+ Text tid = new Text("8");
+ Text per = null;
+
+ ArrayList<KeyExtent> extents = new ArrayList<KeyExtent>();
+
+ for (Long split : splits) {
+ Text er = new Text(String.format("%012d", split));
+ KeyExtent ke = new KeyExtent(tid, er, per);
+ per = er;
+
+ extents.add(ke);
+ }
+
+ extents.add(new KeyExtent(tid, null, per));
+
+ if (args[0].equals("write")) {
+
+ BatchWriter bw = connector.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig());
+
+ for (KeyExtent extent : extents) {
+ Mutation mut = extent.getPrevRowUpdateMutation();
+ new TServerInstance(AddressUtil.parseAddress("192.168.1.100", 4567), "DEADBEEF").putLocation(mut);
+ bw.addMutation(mut);
+ }
+
+ bw.close();
+ } else if (args[0].equals("writeFiles")) {
+ BatchWriter bw = connector.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig());
+
+ for (KeyExtent extent : extents) {
+
+ Mutation mut = new Mutation(extent.getMetadataEntry());
+
+ String dir = "/t-" + UUID.randomUUID();
+
+ Constants.METADATA_DIRECTORY_COLUMN.put(mut, new Value(dir.getBytes(Constants.UTF8)));
+
+ for (int i = 0; i < 5; i++) {
+ mut.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(dir + "/00000_0000" + i + ".map"), new Value("10000,1000000".getBytes(Constants.UTF8)));
+ }
+
+ bw.addMutation(mut);
+ }
+
+ bw.close();
+ } else if (args[0].equals("scan")) {
+
+ int numThreads = Integer.parseInt(args[1]);
+ final int numLoop = Integer.parseInt(args[2]);
+ int numLookups = Integer.parseInt(args[3]);
+
+ HashSet<Integer> indexes = new HashSet<Integer>();
+ while (indexes.size() < numLookups) {
+ indexes.add(r.nextInt(extents.size()));
+ }
+
+ final List<Range> ranges = new ArrayList<Range>();
+ for (Integer i : indexes) {
+ ranges.add(extents.get(i).toMetadataRange());
+ }
+
+ Thread threads[] = new Thread[numThreads];
+
+ for (int i = 0; i < threads.length; i++) {
+ threads[i] = new Thread(new Runnable() {
+
+ @Override
+ public void run() {
+ try {
+ System.out.println(runScanTest(connector, numLoop, ranges));
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ });
+ }
+
+ long t1 = System.currentTimeMillis();
+
+ for (int i = 0; i < threads.length; i++) {
+ threads[i].start();
+ }
+
+ for (int i = 0; i < threads.length; i++) {
+ threads[i].join();
+ }
+
+ long t2 = System.currentTimeMillis();
+
+ System.out.printf("tt : %6.2f%n", (t2 - t1) / 1000.0);
+
+ } else {
+ throw new IllegalArgumentException();
+ }
+
+ }
+
+ private static ScanStats runScanTest(Connector connector, int numLoop, List<Range> ranges) throws Exception {
+ Scanner scanner = null;/*
+ * connector.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS); ColumnFQ.fetch(scanner,
+ * Constants.METADATA_LOCATION_COLUMN); ColumnFQ.fetch(scanner, Constants.METADATA_PREV_ROW_COLUMN);
+ */
+
+ BatchScanner bs = connector.createBatchScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS, 1);
+ bs.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
+ Constants.METADATA_PREV_ROW_COLUMN.fetch(bs);
+
+ bs.setRanges(ranges);
+
+ // System.out.println(ranges);
+
+ ScanStats stats = new ScanStats();
+ for (int i = 0; i < numLoop; i++) {
+ ScanStat ss = scan(bs, ranges, scanner);
+ stats.merge(ss);
+ }
+
+ return stats;
+ }
+
+ private static class ScanStat {
+ long delta1;
+ long delta2;
+ int count1;
+ int count2;
+ }
+
+ private static class ScanStats {
+ Stat delta1 = new Stat();
+ Stat delta2 = new Stat();
+ Stat count1 = new Stat();
+ Stat count2 = new Stat();
+
+ void merge(ScanStat ss) {
+ delta1.addStat(ss.delta1);
+ delta2.addStat(ss.delta2);
+ count1.addStat(ss.count1);
+ count2.addStat(ss.count2);
+ }
+
+ @Override
+ public String toString() {
+ return "[" + delta1 + "] [" + delta2 + "]";
+ }
+ }
+
+ private static ScanStat scan(BatchScanner bs, List<Range> ranges, Scanner scanner) {
+
+ // System.out.println("ranges : "+ranges);
+
+ ScanStat ss = new ScanStat();
+
+ long t1 = System.currentTimeMillis();
+ int count = 0;
+ for (@SuppressWarnings("unused")
+ Entry<Key,Value> entry : bs) {
+ count++;
+ }
+ long t2 = System.currentTimeMillis();
+
+ ss.delta1 = (t2 - t1);
+ ss.count1 = count;
+
+ count = 0;
+ t1 = System.currentTimeMillis();
+ /*
+ * for (Range range : ranges) { scanner.setRange(range); for (Entry<Key, Value> entry : scanner) { count++; } }
+ */
+
+ t2 = System.currentTimeMillis();
+
+ ss.delta2 = (t2 - t1);
+ ss.count2 = count;
+
+ return ss;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/performance/thrift/NullTserver.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/performance/thrift/NullTserver.java
index 71970d3,0000000..e6fcd5b
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/performance/thrift/NullTserver.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/thrift/NullTserver.java
@@@ -1,258 -1,0 +1,252 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.performance.thrift;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
- import org.apache.accumulo.trace.thrift.TInfo;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.thrift.InitialMultiScan;
+import org.apache.accumulo.core.data.thrift.InitialScan;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.data.thrift.MapFileInfo;
+import org.apache.accumulo.core.data.thrift.MultiScanResult;
+import org.apache.accumulo.core.data.thrift.ScanResult;
+import org.apache.accumulo.core.data.thrift.TColumn;
+import org.apache.accumulo.core.data.thrift.TConstraintViolationSummary;
+import org.apache.accumulo.core.data.thrift.TKeyExtent;
+import org.apache.accumulo.core.data.thrift.TMutation;
+import org.apache.accumulo.core.data.thrift.TRange;
+import org.apache.accumulo.core.data.thrift.UpdateErrors;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Processor;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.client.ClientServiceHandler;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.master.state.Assignment;
+import org.apache.accumulo.server.master.state.MetaDataStateStore;
+import org.apache.accumulo.server.master.state.MetaDataTableScanner;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletLocationState;
+import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.accumulo.server.util.TServerUtils;
+import org.apache.accumulo.server.zookeeper.TransactionWatcher;
++import org.apache.accumulo.trace.thrift.TInfo;
+import org.apache.hadoop.io.Text;
+import org.apache.thrift.TException;
+
+import com.beust.jcommander.Parameter;
+
+
+/**
+ * The purpose of this class is to server as fake tserver that is a data sink like /dev/null. NullTserver modifies the !METADATA location entries for a table to
+ * point to it. This allows thrift performance to be measured by running any client code that writes to a table.
+ *
+ */
+
+public class NullTserver {
+
+ public static class ThriftClientHandler extends ClientServiceHandler implements TabletClientService.Iface {
+
+ private long updateSession = 1;
+
+ public ThriftClientHandler(Instance instance, TransactionWatcher watcher) {
+ super(instance, watcher);
+ }
+
+ @Override
+ public long startUpdate(TInfo tinfo, TCredentials credentials) {
+ return updateSession++;
+ }
+
+ @Override
+ public void applyUpdates(TInfo tinfo, long updateID, TKeyExtent keyExtent, List<TMutation> mutation) {}
+
+ @Override
+ public UpdateErrors closeUpdate(TInfo tinfo, long updateID) {
+ return new UpdateErrors(new HashMap<TKeyExtent,Long>(), new ArrayList<TConstraintViolationSummary>(), new HashMap<TKeyExtent, SecurityErrorCode>());
+ }
+
+ @Override
+ public List<TKeyExtent> bulkImport(TInfo tinfo, TCredentials credentials, long tid, Map<TKeyExtent,Map<String,MapFileInfo>> files, boolean setTime) {
+ return null;
+ }
+
+ @Override
+ public void closeMultiScan(TInfo tinfo, long scanID) {}
+
+ @Override
+ public void closeScan(TInfo tinfo, long scanID) {}
+
+ @Override
+ public MultiScanResult continueMultiScan(TInfo tinfo, long scanID) {
+ return null;
+ }
+
+ @Override
+ public ScanResult continueScan(TInfo tinfo, long scanID) {
+ return null;
+ }
+
+ @Override
+ public void splitTablet(TInfo tinfo, TCredentials credentials, TKeyExtent extent, ByteBuffer splitPoint) {
+
+ }
+
+ @Override
+ public InitialMultiScan startMultiScan(TInfo tinfo, TCredentials credentials, Map<TKeyExtent,List<TRange>> batch, List<TColumn> columns,
+ List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites) {
+ return null;
+ }
+
+ @Override
+ public InitialScan startScan(TInfo tinfo, TCredentials credentials, TKeyExtent extent, TRange range, List<TColumn> columns, int batchSize,
+ List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites, boolean isolated) {
+ return null;
+ }
+
+ @Override
+ public void update(TInfo tinfo, TCredentials credentials, TKeyExtent keyExtent, TMutation mutation) {
+
+ }
+
+ @Override
+ public TabletServerStatus getTabletServerStatus(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ return null;
+ }
+
+ @Override
+ public List<TabletStats> getTabletStats(TInfo tinfo, TCredentials credentials, String tableId) throws ThriftSecurityException, TException {
+ return null;
+ }
+
+ @Override
+ public TabletStats getHistoricalStats(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ return null;
+ }
+
+ @Override
+ public void halt(TInfo tinfo, TCredentials credentials, String lock) throws ThriftSecurityException, TException {}
+
+ @Override
+ public void fastHalt(TInfo tinfo, TCredentials credentials, String lock) {}
+
+ @Override
+ public void loadTablet(TInfo tinfo, TCredentials credentials, String lock, TKeyExtent extent) throws TException {}
+
+ @Override
+ public void unloadTablet(TInfo tinfo, TCredentials credentials, String lock, TKeyExtent extent, boolean save) throws TException {}
+
+ @Override
+ public List<ActiveScan> getActiveScans(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ return new ArrayList<ActiveScan>();
+ }
+
+ @Override
+ public void chop(TInfo tinfo, TCredentials credentials, String lock, TKeyExtent extent) throws TException {}
+
+ @Override
+ public void flushTablet(TInfo tinfo, TCredentials credentials, String lock, TKeyExtent extent) throws TException {
+
+ }
+
+ @Override
+ public void compact(TInfo tinfo, TCredentials credentials, String lock, String tableId, ByteBuffer startRow, ByteBuffer endRow) throws TException {
+
+ }
+
+ @Override
+ public void flush(TInfo tinfo, TCredentials credentials, String lock, String tableId, ByteBuffer startRow, ByteBuffer endRow) throws TException {
+
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface#removeLogs(org.apache.accumulo.trace.thrift.TInfo,
- * org.apache.accumulo.core.security.thrift.Credentials, java.util.List)
- */
+ @Override
+ public void removeLogs(TInfo tinfo, TCredentials credentials, List<String> filenames) throws TException {
+ }
+
+ @Override
+ public List<ActiveCompaction> getActiveCompactions(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ return new ArrayList<ActiveCompaction>();
+ }
+ }
+
+ static class Opts extends Help {
+ @Parameter(names={"-i", "--instance"}, description="instance name", required=true)
+ String iname = null;
+ @Parameter(names={"-z", "--keepers"}, description="comma-separated list of zookeeper host:ports", required=true)
+ String keepers = null;
+ @Parameter(names="--table", description="table to adopt", required=true)
+ String tableName = null;
+ @Parameter(names="--port", description="port number to use")
+ int port = DefaultConfiguration.getInstance().getPort(Property.TSERV_CLIENTPORT);
+ }
+
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(NullTserver.class.getName(), args);
+
+ TransactionWatcher watcher = new TransactionWatcher();
+ ThriftClientHandler tch = new ThriftClientHandler(HdfsZooInstance.getInstance(), watcher);
+ Processor<Iface> processor = new Processor<Iface>(tch);
+ TServerUtils.startTServer(opts.port, processor, "NullTServer", "null tserver", 2, 1000, 10*1024*1024);
+
+ InetSocketAddress addr = new InetSocketAddress(InetAddress.getLocalHost(), opts.port);
+
+ // modify !METADATA
+ ZooKeeperInstance zki = new ZooKeeperInstance(opts.iname, opts.keepers);
+ String tableId = Tables.getTableId(zki, opts.tableName);
+
+ // read the locations for the table
+ Range tableRange = new KeyExtent(new Text(tableId), null, null).toMetadataRange();
+ MetaDataTableScanner s = new MetaDataTableScanner(zki, SecurityConstants.getSystemCredentials(), tableRange);
+ long randomSessionID = opts.port;
+ TServerInstance instance = new TServerInstance(addr, randomSessionID);
+ List<Assignment> assignments = new ArrayList<Assignment>();
+ while (s.hasNext()) {
+ TabletLocationState next = s.next();
+ assignments.add(new Assignment(next.extent, instance));
+ }
+ s.close();
+ // point them to this server
+ MetaDataStateStore store = new MetaDataStateStore();
+ store.setLocations(assignments);
+
+ while (true) {
+ UtilWaitThread.sleep(10000);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/randomwalk/Framework.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/Framework.java
index 9d01929,0000000..7cb58c9
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Framework.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Framework.java
@@@ -1,129 -1,0 +1,126 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.randomwalk;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.util.HashMap;
+import java.util.Properties;
+
+import org.apache.log4j.Logger;
+import org.apache.log4j.xml.DOMConfigurator;
+
+import com.beust.jcommander.Parameter;
+
+public class Framework {
+
+ private static final Logger log = Logger.getLogger(Framework.class);
+ private HashMap<String,Node> nodes = new HashMap<String,Node>();
+ private String configDir = null;
+ private static final Framework INSTANCE = new Framework();
+
+ /**
+ * @return Singleton instance of Framework
+ */
+ public static Framework getInstance() {
+ return INSTANCE;
+ }
+
+ public String getConfigDir() {
+ return configDir;
+ }
+
+ public void setConfigDir(String confDir) {
+ configDir = confDir;
+ }
+
+ /**
+ * Run random walk framework
+ *
+ * @param startName
+ * Full name of starting graph or test
- * @param state
- * @param confDir
+ */
+ public int run(String startName, State state, String confDir) {
+
+ try {
+ System.out.println("confDir " + confDir);
+ setConfigDir(confDir);
+ Node node = getNode(startName);
+ node.visit(state, new Properties());
+ } catch (Exception e) {
+ log.error("Error during random walk", e);
+ return -1;
+ }
+ return 0;
+ }
+
+ /**
+ * Creates node (if it does not already exist) and inserts into map
+ *
+ * @param id
+ * Name of node
+ * @return Node specified by id
- * @throws Exception
+ */
+ public Node getNode(String id) throws Exception {
+
+ // check for node in nodes
+ if (nodes.containsKey(id)) {
+ return nodes.get(id);
+ }
+
+ // otherwise create and put in nodes
+ Node node = null;
+ if (id.endsWith(".xml")) {
+ node = new Module(new File(configDir + "modules/" + id));
+ } else {
+ node = (Test) Class.forName(id).newInstance();
+ }
+ nodes.put(id, node);
+ return node;
+ }
+
+ static class Opts extends org.apache.accumulo.core.cli.Help {
+ @Parameter(names="--configDir", required=true, description="directory containing the test configuration")
+ String configDir;
+ @Parameter(names="--logDir", required=true, description="location of the local logging directory")
+ String localLogPath;
+ @Parameter(names="--logId", required=true, description="a unique log identifier (like a hostname, or pid)")
+ String logId;
+ @Parameter(names="--module", required=true, description="the name of the module to run")
+ String module;
+ }
+
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(Framework.class.getName(), args);
+
+ Properties props = new Properties();
+ FileInputStream fis = new FileInputStream(opts.configDir + "/randomwalk.conf");
+ props.load(fis);
+ fis.close();
+
+ System.setProperty("localLog", opts.localLogPath + "/" + opts.logId);
+ System.setProperty("nfsLog", props.getProperty("NFS_LOGPATH") + "/" + opts.logId);
+
+ DOMConfigurator.configure(opts.configDir + "logger.xml");
+
+ State state = new State(props);
+ int retval = getInstance().run(opts.module, state, opts.configDir);
+
+ System.exit(retval);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
index 1868ade,0000000..b74b6cd
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
@@@ -1,64 -1,0 +1,63 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.randomwalk;
+
+import java.util.Properties;
+
+import org.apache.log4j.Logger;
+
+/**
+ * Represents a point in graph of RandomFramework
+ */
+public abstract class Node {
+
+ protected final Logger log = Logger.getLogger(this.getClass());
+ long progress = System.currentTimeMillis();
+
+ /**
+ * Visits node
+ *
+ * @param state
+ * Random walk state passed between nodes
- * @throws Exception
+ */
+ public abstract void visit(State state, Properties props) throws Exception;
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null)
+ return false;
+ return toString().equals(o.toString());
+ }
+
+ @Override
+ public String toString() {
+ return this.getClass().getName();
+ }
+
+ @Override
+ public int hashCode() {
+ return toString().hashCode();
+ }
+
+ synchronized public void makingProgress() {
+ progress = System.currentTimeMillis();
+ }
+
+ synchronized public long lastProgress() {
+ return progress;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
index a0dd37c,0000000..4581b04
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
@@@ -1,110 -1,0 +1,107 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.randomwalk.concurrent;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.test.randomwalk.State;
+import org.apache.accumulo.test.randomwalk.Test;
+
+/**
+ *
+ */
+public class CheckBalance extends Test {
+
+ static final String LAST_UNBALANCED_TIME = "lastUnbalancedTime";
+ static final String UNBALANCED_COUNT = "unbalancedCount";
+
- /* (non-Javadoc)
- * @see org.apache.accumulo.test.randomwalk.Node#visit(org.apache.accumulo.test.randomwalk.State, java.util.Properties)
- */
+ @Override
+ public void visit(State state, Properties props) throws Exception {
+ log.debug("checking balance");
+ Map<String,Long> counts = new HashMap<String,Long>();
+ Scanner scanner = state.getConnector().createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ scanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
+ for (Entry<Key,Value> entry : scanner) {
+ String location = entry.getKey().getColumnQualifier().toString();
+ Long count = counts.get(location);
+ if (count == null)
+ count = Long.valueOf(0);
+ counts.put(location, count + 1);
+ }
+ double total = 0.;
+ for (Long count : counts.values()) {
+ total += count.longValue();
+ }
+ final double average = total / counts.size();
+ final double sd = stddev(counts.values(), average);
+ log.debug("average " + average + ", standard deviation " + sd);
+
+ // Check for balanced # of tablets on each node
+ double maxDifference = 2.0 * sd;
+ String unbalancedLocation = null;
+ long lastCount = 0L;
+ boolean balanced = true;
+ for (Entry<String,Long> entry : counts.entrySet()) {
+ long thisCount = entry.getValue().longValue();
+ if (Math.abs(thisCount - average) > maxDifference) {
+ balanced = false;
+ log.debug("unbalanced: " + entry.getKey() + " has " + entry.getValue() + " tablets and the average is " + average);
+ unbalancedLocation = entry.getKey();
+ lastCount = thisCount;
+ }
+ }
+
+ // It is expected that the number of tablets will be uneven for short
+ // periods of time. Don't complain unless we've seen it only unbalanced
+ // over a 15 minute period and it's been at least three checks.
+ if (!balanced) {
+ Long last = state.getLong(LAST_UNBALANCED_TIME);
+ if (last != null && System.currentTimeMillis() - last > 15 * 60 * 1000) {
+ Integer count = state.getInteger(UNBALANCED_COUNT);
+ if (count == null)
+ count = Integer.valueOf(0);
+ if (count > 3)
+ throw new Exception("servers are unbalanced! location " + unbalancedLocation + " count " + lastCount + " too far from average " + average);
+ count++;
+ state.set(UNBALANCED_COUNT, count);
+ }
+ if (last == null)
+ state.set(LAST_UNBALANCED_TIME, System.currentTimeMillis());
+ } else {
+ state.remove(LAST_UNBALANCED_TIME);
+ state.remove(UNBALANCED_COUNT);
+ }
+ }
+
+ private static double stddev(Collection<Long> samples, double avg) {
+ int num = samples.size();
+ double sqrtotal = 0.0;
+ for (Long s : samples) {
+ double diff = s.doubleValue() - avg;
+ sqrtotal += diff * diff;
+ }
- return Math.sqrt(sqrtotal / (double) num);
++ return Math.sqrt(sqrtotal / num);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
----------------------------------------------------------------------
diff --cc trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
index 049b2a2,0000000..dfa9f0c
mode 100644,000000..100644
--- a/trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
+++ b/trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
@@@ -1,132 -1,0 +1,122 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.trace.instrument.receivers;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.ZooKeeper.States;
+
+/**
+ * Find a Span collector via zookeeper and push spans there via Thrift RPC
+ *
+ */
+public class ZooSpanClient extends SendSpansViaThrift {
+
+ private static final Logger log = Logger.getLogger(ZooSpanClient.class);
+ private static final int TOTAL_TIME_WAIT_CONNECT_MS = 10 * 1000;
+ private static final int TIME_WAIT_CONNECT_CHECK_MS = 100;
+ private static final Charset UTF8 = Charset.forName("UTF-8");
+
+ ZooKeeper zoo = null;
+ final String path;
+ final Random random = new Random();
+ final List<String> hosts = new ArrayList<String>();
+
+ public ZooSpanClient(String keepers, final String path, String host, String service, long millis) throws IOException, KeeperException, InterruptedException {
+ super(host, service, millis);
+ this.path = path;
+ zoo = new ZooKeeper(keepers, 30 * 1000, new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ try {
+ if (zoo != null) {
+ updateHosts(path, zoo.getChildren(path, null));
+ }
+ } catch (Exception ex) {
+ log.error("unable to get destination hosts in zookeeper", ex);
+ }
+ }
+ });
+ for (int i = 0; i < TOTAL_TIME_WAIT_CONNECT_MS; i += TIME_WAIT_CONNECT_CHECK_MS) {
+ if (zoo.getState().equals(States.CONNECTED))
+ break;
+ try {
+ Thread.sleep(TIME_WAIT_CONNECT_CHECK_MS);
+ } catch (InterruptedException ex) {
+ break;
+ }
+ }
+ zoo.getChildren(path, true);
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see trace.instrument.receivers.AsyncSpanReceiver#flush()
- */
+ @Override
+ public void flush() {
+ if (!hosts.isEmpty())
+ super.flush();
+ }
+
- /*
- * (non-Javadoc)
- *
- * @see trace.instrument.receivers.AsyncSpanReceiver#sendSpans()
- */
+ @Override
+ void sendSpans() {
+ if (hosts.isEmpty()) {
+ if (!sendQueue.isEmpty()) {
+ log.error("No hosts to send data to, dropping queued spans");
+ synchronized (sendQueue) {
+ sendQueue.clear();
+ sendQueue.notifyAll();
+ }
+ }
+ } else {
+ super.sendSpans();
+ }
+ }
+
+ synchronized private void updateHosts(String path, List<String> children) {
+ log.debug("Scanning trace hosts in zookeeper: " + path);
+ try {
+ List<String> hosts = new ArrayList<String>();
+ for (String child : children) {
+ byte[] data = zoo.getData(path + "/" + child, null, null);
+ hosts.add(new String(data, UTF8));
+ }
+ this.hosts.clear();
+ this.hosts.addAll(hosts);
+ log.debug("Trace hosts: " + this.hosts);
+ } catch (Exception ex) {
+ log.error("unable to get destination hosts in zookeeper", ex);
+ }
+ }
+
+ @Override
+ synchronized protected String getSpanKey(Map<String,String> data) {
+ if (hosts.size() > 0) {
+ String host = hosts.get(random.nextInt(hosts.size()));
+ log.debug("sending data to " + host);
+ return host;
+ }
+ return null;
+ }
+}
[36/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
index efadfae,0000000..09fbbd2
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
@@@ -1,279 -1,0 +1,278 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
+import java.net.Socket;
+import java.net.URLEncoder;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.net.SocketFactory;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.commons.io.filefilter.WildcardFileFilter;
+import org.apache.commons.lang.math.LongRange;
+import org.apache.log4j.Category;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.Filter;
+import org.apache.log4j.spi.LocationInfo;
+import org.apache.log4j.spi.LoggingEvent;
+import org.apache.log4j.spi.ThrowableInformation;
+import org.apache.log4j.varia.LevelRangeFilter;
+import org.apache.log4j.xml.XMLLayout;
+
+import com.beust.jcommander.IStringConverter;
+import com.beust.jcommander.Parameter;
+
+public class SendLogToChainsaw extends XMLLayout {
+
+ private static Pattern logPattern = Pattern.compile(
+ "^(\\d\\d)\\s(\\d\\d):(\\d\\d):(\\d\\d),(\\d\\d\\d)\\s\\[(.*)\\]\\s(TRACE|DEBUG|INFO|WARN|FATAL|ERROR)\\s*?:(.*)$", Pattern.UNIX_LINES);
+
+ private File[] logFiles = null;
+
+ private SocketFactory factory = SocketFactory.getDefault();
+
+ private WildcardFileFilter fileFilter = null;
+
+ private Socket socket = null;
+
+ private Pattern lineFilter = null;
+
+ private LongRange dateFilter = null;
+
+ private LevelRangeFilter levelFilter = null;
+
+ public SendLogToChainsaw(String directory, String fileNameFilter, String host, int port, Date start, Date end, String regex, String level) throws Exception {
+
+ // Set up the file name filter
+ if (null != fileNameFilter) {
+ fileFilter = new WildcardFileFilter(fileNameFilter);
+ } else {
+ fileFilter = new WildcardFileFilter("*");
+ }
+
+ // Get the list of files that match
+ File dir = new File(directory);
+ if (dir.isDirectory()) {
+ logFiles = dir.listFiles((FilenameFilter) fileFilter);
+ } else {
+ throw new IllegalArgumentException(directory + " is not a directory or is not readable.");
+ }
+
+ if (logFiles.length == 0) {
+ throw new IllegalArgumentException("No files match the supplied filter.");
+ }
+
+ socket = factory.createSocket(host, port);
+
+ lineFilter = Pattern.compile(regex);
+
+ // Create Date Filter
+ if (null != start) {
+ if (end == null)
+ end = new Date(System.currentTimeMillis());
+ dateFilter = new LongRange(start.getTime(), end.getTime());
+ }
+
+ if (null != level) {
+ Level base = Level.toLevel(level.toUpperCase());
+ levelFilter = new LevelRangeFilter();
+ levelFilter.setAcceptOnMatch(true);
+ levelFilter.setLevelMin(base);
+ levelFilter.setLevelMax(Level.FATAL);
+ }
+ }
+
+ public void processLogFiles() throws IOException {
+ String line = null;
+ String out = null;
+ InputStreamReader isReader = null;
+ BufferedReader reader = null;
+ try {
+ for (File log : logFiles) {
+ // Parse the server type and name from the log file name
+ String threadName = log.getName().substring(0, log.getName().indexOf("."));
+ try {
+ isReader = new InputStreamReader(new FileInputStream(log), Constants.UTF8);
+ } catch (FileNotFoundException e) {
+ System.out.println("Unable to find file: " + log.getAbsolutePath());
+ throw e;
+ }
+ reader = new BufferedReader(isReader);
+
+ try {
+ line = reader.readLine();
+ while (null != line) {
+ out = convertLine(line, threadName);
+ if (null != out) {
+ if (socket != null && socket.isConnected())
+ socket.getOutputStream().write(out.getBytes(Constants.UTF8));
+ else
+ System.err.println("Unable to send data to transport");
+ }
+ line = reader.readLine();
+ }
+ } catch (IOException e) {
+ System.out.println("Error processing line: " + line + ". Output was " + out);
+ throw e;
+ } finally {
+ if (reader != null) {
+ reader.close();
+ }
+ if (isReader != null) {
+ isReader.close();
+ }
+ }
+ }
+ } finally {
+ if (socket != null && socket.isConnected()) {
+ socket.close();
+ }
+ }
+ }
+
+ private String convertLine(String line, String threadName) throws UnsupportedEncodingException {
+ String result = null;
+ Matcher m = logPattern.matcher(line);
+ if (m.matches()) {
+
+ Calendar cal = Calendar.getInstance();
+ cal.setTime(new Date(System.currentTimeMillis()));
+ Integer date = Integer.parseInt(m.group(1));
+ Integer hour = Integer.parseInt(m.group(2));
+ Integer min = Integer.parseInt(m.group(3));
+ Integer sec = Integer.parseInt(m.group(4));
+ Integer ms = Integer.parseInt(m.group(5));
+ String clazz = m.group(6);
+ String level = m.group(7);
+ String message = m.group(8);
+ // Apply the regex filter if supplied
+ if (null != lineFilter) {
+ Matcher match = lineFilter.matcher(message);
+ if (!match.matches())
+ return null;
+ }
+ // URL encode the message
+ message = URLEncoder.encode(message, "UTF-8");
+ // Assume that we are processing logs from today.
+ // If the date in the line is greater than today, then it must be
+ // from the previous month.
+ cal.set(Calendar.DATE, date);
+ cal.set(Calendar.HOUR_OF_DAY, hour);
+ cal.set(Calendar.MINUTE, min);
+ cal.set(Calendar.SECOND, sec);
+ cal.set(Calendar.MILLISECOND, ms);
+ if (date > cal.get(Calendar.DAY_OF_MONTH)) {
+ cal.add(Calendar.MONTH, -1);
+ }
+ long ts = cal.getTimeInMillis();
+ // If this event is not between the start and end dates, then skip it.
+ if (null != dateFilter && !dateFilter.containsLong(ts))
+ return null;
+ Category c = Logger.getLogger(clazz);
+ Level l = Level.toLevel(level);
+ LoggingEvent event = new LoggingEvent(clazz, c, ts, l, message, threadName, (ThrowableInformation) null, (String) null, (LocationInfo) null,
+ (Map<?,?>) null);
+ // Check the log level filter
+ if (null != levelFilter && (levelFilter.decide(event) == Filter.DENY)) {
+ return null;
+ }
+ result = format(event);
+ }
+ return result;
+ }
+
+ private static class DateConverter implements IStringConverter<Date> {
+ @Override
+ public Date convert(String value) {
+ SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
+ try {
+ return formatter.parse(value);
+ } catch (ParseException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+ private static class Opts extends Help {
+
+ @Parameter(names={"-d", "--logDirectory"}, description="ACCUMULO log directory path", required=true)
+ String dir;
+
+ @Parameter(names={"-f", "--fileFilter"}, description="filter to apply to names of logs")
+ String filter;
+
+ @Parameter(names={"-h", "--host"}, description="host where chainsaw is running", required=true)
+ String hostname;
+
+ @Parameter(names={"-p", "--port"}, description="port where XMLSocketReceiver is listening", required=true)
+ int portnum;
+
+ @Parameter(names={"-s", "--start"}, description="start date filter (yyyyMMddHHmmss)", required=true, converter=DateConverter.class)
+ Date startDate;
+
+ @Parameter(names={"-e", "--end"}, description="end date filter (yyyyMMddHHmmss)", required=true, converter=DateConverter.class)
+ Date endDate;
+
+ @Parameter(names={"-l", "--level"}, description="filter log level")
+ String level;
+
+ @Parameter(names={"-m", "--messageFilter"}, description="regex filter for log messages")
+ String regex;
+ }
+
+
+
+
+ /**
+ *
+ * @param args
+ * 0: path to log directory parameter
+ * 1: filter to apply for logs to include (uses wildcards (i.e. logger* and IS case sensitive) parameter
+ * 2: chainsaw host parameter
+ * 3: chainsaw port parameter
+ * 4: start date filter parameter
+ * 5: end date filter parameter
+ * 6: optional regex filter to match on each log4j message parameter
+ * 7: optional level filter
- * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(SendLogToChainsaw.class.getName(), args);
+
+ SendLogToChainsaw c = new SendLogToChainsaw(opts.dir, opts.filter, opts.hostname, opts.portnum, opts.startDate, opts.endDate, opts.regex, opts.level);
+ c.processLogFiles();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/TServerUtils.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/TServerUtils.java
index 50476a2,0000000..fa4de30
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/TServerUtils.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/TServerUtils.java
@@@ -1,313 -1,0 +1,314 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.UnknownHostException;
+import java.nio.channels.ServerSocketChannel;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.LoggingRunnable;
+import org.apache.accumulo.core.util.SimpleThreadPool;
+import org.apache.accumulo.core.util.TBufferedSocket;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.thrift.metrics.ThriftMetrics;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+import org.apache.thrift.TProcessor;
+import org.apache.thrift.TProcessorFactory;
+import org.apache.thrift.protocol.TProtocol;
+import org.apache.thrift.server.TServer;
+import org.apache.thrift.server.TThreadPoolServer;
+import org.apache.thrift.transport.TNonblockingSocket;
+import org.apache.thrift.transport.TServerTransport;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+
+public class TServerUtils {
+ private static final Logger log = Logger.getLogger(TServerUtils.class);
+
+ public static final ThreadLocal<String> clientAddress = new ThreadLocal<String>();
+
+ public static class ServerPort {
+ public final TServer server;
+ public final int port;
+
+ public ServerPort(TServer server, int port) {
+ this.server = server;
+ this.port = port;
+ }
+ }
+
+ /**
+ * Start a server, at the given port, or higher, if that port is not available.
+ *
+ * @param portHintProperty
+ * the port to attempt to open, can be zero, meaning "any available port"
+ * @param processor
+ * the service to be started
+ * @param serverName
+ * the name of the class that is providing the service
+ * @param threadName
+ * name this service's thread for better debugging
- * @param portSearchProperty
- * @param minThreadProperty
- * @param timeBetweenThreadChecksProperty
+ * @return the server object created, and the port actually used
+ * @throws UnknownHostException
+ * when we don't know our own address
+ */
+ public static ServerPort startServer(AccumuloConfiguration conf, Property portHintProperty, TProcessor processor, String serverName, String threadName,
+ Property portSearchProperty,
+ Property minThreadProperty,
+ Property timeBetweenThreadChecksProperty,
+ Property maxMessageSizeProperty) throws UnknownHostException {
+ int portHint = conf.getPort(portHintProperty);
+ int minThreads = 2;
+ if (minThreadProperty != null)
+ minThreads = conf.getCount(minThreadProperty);
+ long timeBetweenThreadChecks = 1000;
+ if (timeBetweenThreadChecksProperty != null)
+ timeBetweenThreadChecks = conf.getTimeInMillis(timeBetweenThreadChecksProperty);
+ long maxMessageSize = 10 * 1000 * 1000;
+ if (maxMessageSizeProperty != null)
+ maxMessageSize = conf.getMemoryInBytes(maxMessageSizeProperty);
+ boolean portSearch = false;
+ if (portSearchProperty != null)
+ portSearch = conf.getBoolean(portSearchProperty);
+ Random random = new Random();
+ for (int j = 0; j < 100; j++) {
+
+ // Are we going to slide around, looking for an open port?
+ int portsToSearch = 1;
+ if (portSearch)
+ portsToSearch = 1000;
+
+ for (int i = 0; i < portsToSearch; i++) {
+ int port = portHint + i;
+ if (portHint != 0 && i > 0)
+ port = 1024 + random.nextInt(65535 - 1024);
+ if (port > 65535)
+ port = 1024 + port % (65535 - 1024);
+ try {
+ return TServerUtils.startTServer(port, processor, serverName, threadName, minThreads, timeBetweenThreadChecks, maxMessageSize);
+ } catch (Exception ex) {
+ log.info("Unable to use port " + port + ", retrying. (Thread Name = " + threadName + ")");
+ UtilWaitThread.sleep(250);
+ }
+ }
+ }
+ throw new UnknownHostException("Unable to find a listen port");
+ }
+
+ public static class TimedProcessor implements TProcessor {
+
+ final TProcessor other;
+ ThriftMetrics metrics = null;
+ long idleStart = 0;
+
+ TimedProcessor(TProcessor next, String serverName, String threadName) {
+ this.other = next;
+ // Register the metrics MBean
+ try {
+ metrics = new ThriftMetrics(serverName, threadName);
+ metrics.register();
+ } catch (Exception e) {
+ log.error("Exception registering MBean with MBean Server", e);
+ }
+ idleStart = System.currentTimeMillis();
+ }
+
+ @Override
+ public boolean process(TProtocol in, TProtocol out) throws TException {
+ long now = 0;
+ if (metrics.isEnabled()) {
+ now = System.currentTimeMillis();
+ metrics.add(ThriftMetrics.idle, (now - idleStart));
+ }
+ try {
+ try {
+ return other.process(in, out);
+ } catch (NullPointerException ex) {
+ // THRIFT-1447 - remove with thrift 0.9
+ return true;
+ }
+ } finally {
+ if (metrics.isEnabled()) {
+ idleStart = System.currentTimeMillis();
+ metrics.add(ThriftMetrics.execute, idleStart - now);
+ }
+ }
+ }
+ }
+
+ public static class ClientInfoProcessorFactory extends TProcessorFactory {
+
+ public ClientInfoProcessorFactory(TProcessor processor) {
+ super(processor);
+ }
+
++ @Override
+ public TProcessor getProcessor(TTransport trans) {
+ if (trans instanceof TBufferedSocket) {
+ TBufferedSocket tsock = (TBufferedSocket) trans;
+ clientAddress.set(tsock.getClientString());
+ }
+ return super.getProcessor(trans);
+ }
+ }
+
+ public static class THsHaServer extends org.apache.thrift.server.THsHaServer {
+ public THsHaServer(Args args) {
+ super(args);
+ }
+
++ @Override
+ protected Runnable getRunnable(FrameBuffer frameBuffer) {
+ return new Invocation(frameBuffer);
+ }
+
+ private class Invocation implements Runnable {
+
+ private final FrameBuffer frameBuffer;
+
+ public Invocation(final FrameBuffer frameBuffer) {
+ this.frameBuffer = frameBuffer;
+ }
+
++ @Override
+ public void run() {
+ if (frameBuffer.trans_ instanceof TNonblockingSocket) {
+ TNonblockingSocket tsock = (TNonblockingSocket) frameBuffer.trans_;
+ Socket sock = tsock.getSocketChannel().socket();
+ clientAddress.set(sock.getInetAddress().getHostAddress() + ":" + sock.getPort());
+ }
+ frameBuffer.invoke();
+ }
+ }
+ }
+
+ public static ServerPort startHsHaServer(int port, TProcessor processor, final String serverName, String threadName, final int numThreads,
+ long timeBetweenThreadChecks, long maxMessageSize) throws TTransportException {
+ TNonblockingServerSocket transport = new TNonblockingServerSocket(port);
+ if (port == 0) {
+ port = transport.getPort();
+ }
+ THsHaServer.Args options = new THsHaServer.Args(transport);
+ options.protocolFactory(ThriftUtil.protocolFactory());
+ options.transportFactory(ThriftUtil.transportFactory(maxMessageSize));
+ options.maxReadBufferBytes = maxMessageSize;
+ options.stopTimeoutVal(5);
+ /*
+ * Create our own very special thread pool.
+ */
+ final ThreadPoolExecutor pool = new SimpleThreadPool(numThreads, "ClientPool");
+ // periodically adjust the number of threads we need by checking how busy our threads are
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ if (pool.getCorePoolSize() <= pool.getActiveCount()) {
+ int larger = pool.getCorePoolSize() + Math.min(pool.getQueue().size(), 2);
+ log.info("Increasing server thread pool size on " + serverName + " to " + larger);
+ pool.setMaximumPoolSize(larger);
+ pool.setCorePoolSize(larger);
+ } else {
+ if (pool.getCorePoolSize() > pool.getActiveCount() + 3) {
+ int smaller = Math.max(numThreads, pool.getCorePoolSize() - 1);
+ if (smaller != pool.getCorePoolSize()) {
+ // there is a race condition here... the active count could be higher by the time
+ // we decrease the core pool size... so the active count could end up higher than
+ // the core pool size, in which case everything will be queued... the increase case
+ // should handle this and prevent deadlock
+ log.info("Decreasing server thread pool size on " + serverName + " to " + smaller);
+ pool.setCorePoolSize(smaller);
+ }
+ }
+ }
+ }
+ }, timeBetweenThreadChecks, timeBetweenThreadChecks);
+ options.executorService(pool);
+ processor = new TServerUtils.TimedProcessor(processor, serverName, threadName);
+ options.processorFactory(new TProcessorFactory(processor));
+ return new ServerPort(new THsHaServer(options), port);
+ }
+
+ public static ServerPort startThreadPoolServer(int port, TProcessor processor, String serverName, String threadName, int numThreads)
+ throws TTransportException {
+
+ // if port is zero, then we must bind to get the port number
+ ServerSocket sock;
+ try {
+ sock = ServerSocketChannel.open().socket();
+ sock.setReuseAddress(true);
+ sock.bind(new InetSocketAddress(port));
+ port = sock.getLocalPort();
+ } catch (IOException ex) {
+ throw new TTransportException(ex);
+ }
+ TServerTransport transport = new TBufferedServerSocket(sock, 32 * 1024);
+ TThreadPoolServer.Args options = new TThreadPoolServer.Args(transport);
+ options.protocolFactory(ThriftUtil.protocolFactory());
+ options.transportFactory(ThriftUtil.transportFactory());
+ processor = new TServerUtils.TimedProcessor(processor, serverName, threadName);
+ options.processorFactory(new ClientInfoProcessorFactory(processor));
+ return new ServerPort(new TThreadPoolServer(options), port);
+ }
+
+ public static ServerPort startTServer(int port, TProcessor processor, String serverName, String threadName, int numThreads, long timeBetweenThreadChecks, long maxMessageSize)
+ throws TTransportException {
+ ServerPort result = startHsHaServer(port, processor, serverName, threadName, numThreads, timeBetweenThreadChecks, maxMessageSize);
+ // ServerPort result = startThreadPoolServer(port, processor, serverName, threadName, -1);
+ final TServer finalServer = result.server;
+ Runnable serveTask = new Runnable() {
++ @Override
+ public void run() {
+ try {
+ finalServer.serve();
+ } catch (Error e) {
+ Halt.halt("Unexpected error in TThreadPoolServer " + e + ", halting.");
+ }
+ }
+ };
+ serveTask = new LoggingRunnable(TServerUtils.log, serveTask);
+ Thread thread = new Daemon(serveTask, threadName);
+ thread.start();
+ return result;
+ }
+
+ // Existing connections will keep our thread running: reach in with reflection and insist that they shutdown.
+ public static void stopTServer(TServer s) {
+ if (s == null)
+ return;
+ s.stop();
+ try {
+ Field f = s.getClass().getDeclaredField("executorService_");
+ f.setAccessible(true);
+ ExecutorService es = (ExecutorService) f.get(s);
+ es.shutdownNow();
+ } catch (Exception e) {
+ TServerUtils.log.error("Unable to call shutdownNow", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
index c3f4a72,0000000..92e0674
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
@@@ -1,48 -1,0 +1,45 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+
+import com.beust.jcommander.Parameter;
+
+public class TableDiskUsage {
+
+ static class Opts extends ClientOpts {
+ @Parameter(description=" <table> { <table> ... } ")
+ List<String> tables = new ArrayList<String>();
+ }
+
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+ FileSystem fs = FileSystem.get(new Configuration());
+ Opts opts = new Opts();
+ opts.parseArgs(TableDiskUsage.class.getName(), args);
+ Connector conn = opts.getConnector();
+ org.apache.accumulo.core.util.TableDiskUsage.printDiskUsage(DefaultConfiguration.getInstance(), opts.tables, fs, conn, false);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/server/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
index 2fc0bd3,0000000..34c2151
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
@@@ -1,75 -1,0 +1,72 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.util.List;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+
+import com.beust.jcommander.Parameter;
+
+public class TabletServerLocks {
+
+ static class Opts extends Help {
+ @Parameter(names="-list")
+ boolean list = false;
+ @Parameter(names="-delete")
+ String delete = null;
+ }
- /**
- * @param args
- */
+ public static void main(String[] args) throws Exception {
+
+ Instance instance = HdfsZooInstance.getInstance();
+ String tserverPath = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
+ Opts opts = new Opts();
+ opts.parseArgs(TabletServerLocks.class.getName(), args);
+
+ ZooCache cache = new ZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
+
+ if (opts.list) {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ List<String> tabletServers = zoo.getChildren(tserverPath);
+
+ for (String tabletServer : tabletServers) {
+ byte[] lockData = ZooLock.getLockData(cache, tserverPath + "/" + tabletServer, null);
+ String holder = null;
+ if (lockData != null) {
+ holder = new String(lockData, Constants.UTF8);
+ }
+
+ System.out.printf("%32s %16s%n", tabletServer, holder);
+ }
+ } else if (opts.delete != null) {
+ ZooLock.deleteLock(tserverPath + "/" + args[1]);
+ } else {
+ System.out.println("Usage : " + TabletServerLocks.class.getName() + " -list|-delete <tserver lock>");
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
----------------------------------------------------------------------
diff --cc start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
index 5f7fd5e,0000000..1e2b4b5
mode 100644,000000..100644
--- a/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
+++ b/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
@@@ -1,255 -1,0 +1,252 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.start.classloader;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import org.apache.log4j.Logger;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+/**
+ *
+ */
+public class AccumuloClassLoader {
+
+ public static final String CLASSPATH_PROPERTY_NAME = "general.classpaths";
+
+ public static final String ACCUMULO_CLASSPATH_VALUE =
+ "$ACCUMULO_CONF_DIR,\n" +
+ "$ACCUMULO_HOME/lib/[^.].*.jar,\n" +
+ "$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n" +
+ "$HADOOP_CONF_DIR,\n" +
+ "$HADOOP_PREFIX/[^.].*.jar,\n" +
+ "$HADOOP_PREFIX/lib/[^.].*.jar,\n" +
+ "$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n" +
+ "$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n" +
+ "$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n" +
+ "$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n" +
+ "/usr/lib/hadoop/[^.].*.jar,\n" +
+ "/usr/lib/hadoop/lib/[^.].*.jar,\n" +
+ "/usr/lib/hadoop-hdfs/[^.].*.jar,\n" +
+ "/usr/lib/hadoop-mapreduce/[^.].*.jar,\n" +
+ "/usr/lib/hadoop-yarn/[^.].*.jar,\n"
+ ;
+
+ private static String SITE_CONF;
+
+ private static URLClassLoader classloader;
+
+ private static Logger log = Logger.getLogger(AccumuloClassLoader.class);
+
+ static {
+ String configFile = System.getProperty("org.apache.accumulo.config.file", "accumulo-site.xml");
+ if (System.getenv("ACCUMULO_CONF_DIR") != null) {
+ // accumulo conf dir should be set
+ SITE_CONF = System.getenv("ACCUMULO_CONF_DIR") + "/" + configFile;
+ } else if (System.getenv("ACCUMULO_HOME") != null) {
+ // if no accumulo conf dir, try accumulo home default
+ SITE_CONF = System.getenv("ACCUMULO_HOME") + "/conf/" + configFile;
+ } else {
+ SITE_CONF = null;
+ }
+ }
+
+ /**
+ * Parses and XML Document for a property node for a <name> with the value propertyName if it finds one the function return that property's value for its
+ * <value> node. If not found the function will return null
+ *
+ * @param d
+ * XMLDocument to search through
+ * @param propertyName
+ */
+ private static String getAccumuloClassPathStrings(Document d, String propertyName) {
+ NodeList pnodes = d.getElementsByTagName("property");
+ for (int i = pnodes.getLength() - 1; i >= 0; i--) {
+ Element current_property = (Element) pnodes.item(i);
+ Node cname = current_property.getElementsByTagName("name").item(0);
+ if (cname != null && cname.getTextContent().compareTo(propertyName) == 0) {
+ Node cvalue = current_property.getElementsByTagName("value").item(0);
+ if (cvalue != null) {
+ return cvalue.getTextContent();
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Looks for the site configuration file for Accumulo and if it has a property for propertyName return it otherwise returns defaultValue Should throw an
+ * exception if the default configuration can not be read;
+ *
+ * @param propertyName
+ * Name of the property to pull
+ * @param defaultValue
+ * Value to default to if not found.
+ * @return site or default class path String
+ */
+
+ public static String getAccumuloString(String propertyName, String defaultValue) {
+
+ try {
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ String site_classpath_string = null;
+ try {
+ Document site_conf = db.parse(SITE_CONF);
+ site_classpath_string = getAccumuloClassPathStrings(site_conf, propertyName);
+ } catch (Exception e) {
+ /* we don't care because this is optional and we can use defaults */
+ }
+ if (site_classpath_string != null)
+ return site_classpath_string;
+ return defaultValue;
+ } catch (Exception e) {
+ throw new IllegalStateException("ClassPath Strings Lookup failed", e);
+ }
+ }
+
+ /**
+ * Replace environment variables in the classpath string with their actual value
- *
- * @param classpath
- * @param env
+ */
+ public static String replaceEnvVars(String classpath, Map<String,String> env) {
+ Pattern envPat = Pattern.compile("\\$[A-Za-z][a-zA-Z0-9_]*");
+ Matcher envMatcher = envPat.matcher(classpath);
+ while (envMatcher.find(0)) {
+ // name comes after the '$'
+ String varName = envMatcher.group().substring(1);
+ String varValue = env.get(varName);
+ if (varValue == null) {
+ varValue = "";
+ }
+ classpath = (classpath.substring(0, envMatcher.start()) + varValue + classpath.substring(envMatcher.end()));
+ envMatcher.reset(classpath);
+ }
+ return classpath;
+ }
+
+ /**
+ * Populate the list of URLs with the items in the classpath string
+ *
+ * @param classpath
+ * @param urls
+ * @throws MalformedURLException
+ */
+ private static void addUrl(String classpath, ArrayList<URL> urls) throws MalformedURLException {
+ classpath = classpath.trim();
+ if (classpath.length() == 0)
+ return;
+
+ classpath = replaceEnvVars(classpath, System.getenv());
+
+ // Try to make a URI out of the classpath
+ URI uri = null;
+ try {
+ uri = new URI(classpath);
+ } catch (URISyntaxException e) {
+ // Not a valid URI
+ }
+
+ if (null == uri || !uri.isAbsolute() || (null != uri.getScheme() && uri.getScheme().equals("file://"))) {
+ // Then treat this URI as a File.
+ // This checks to see if the url string is a dir if it expand and get all jars in that directory
+ final File extDir = new File(classpath);
+ if (extDir.isDirectory())
+ urls.add(extDir.toURI().toURL());
+ else {
+ if (extDir.getParentFile() != null) {
+ File[] extJars = extDir.getParentFile().listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return name.matches("^" + extDir.getName());
+ }
+ });
+ if (extJars != null && extJars.length > 0) {
+ for (File jar : extJars)
+ urls.add(jar.toURI().toURL());
+ } else {
+ log.debug("ignoring classpath entry " + classpath);
+ }
+ } else {
+ log.debug("ignoring classpath entry " + classpath);
+ }
+ }
+ } else {
+ urls.add(uri.toURL());
+ }
+
+ }
+
+ private static ArrayList<URL> findAccumuloURLs() throws IOException {
+ String cp = getAccumuloString(AccumuloClassLoader.CLASSPATH_PROPERTY_NAME, AccumuloClassLoader.ACCUMULO_CLASSPATH_VALUE);
+ if (cp == null)
+ return new ArrayList<URL>();
+ String[] cps = replaceEnvVars(cp, System.getenv()).split(",");
+ ArrayList<URL> urls = new ArrayList<URL>();
+ for (String classpath : cps) {
+ if (!classpath.startsWith("#")) {
+ addUrl(classpath, urls);
+ }
+ }
+ return urls;
+ }
+
+ public static synchronized ClassLoader getClassLoader() throws IOException {
+ if (classloader == null) {
+ ArrayList<URL> urls = findAccumuloURLs();
+
+ ClassLoader parentClassLoader = AccumuloClassLoader.class.getClassLoader();
+
+ log.debug("Create 2nd tier ClassLoader using URLs: " + urls.toString());
+ URLClassLoader aClassLoader = new URLClassLoader(urls.toArray(new URL[urls.size()]), parentClassLoader) {
+ @Override
+ protected synchronized Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
+
+ if (name.startsWith("org.apache.accumulo.start.classloader.vfs")) {
+ Class<?> c = findLoadedClass(name);
+ if (c == null) {
+ try {
+ // try finding this class here instead of parent
+ findClass(name);
+ } catch (ClassNotFoundException e) {}
+ }
+ }
+ return super.loadClass(name, resolve);
+ }
+ };
+ classloader = aClassLoader;
+ }
+
+ return classloader;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
----------------------------------------------------------------------
diff --cc start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
index 7742cbe,0000000..e1ff55e
mode 100644,000000..100644
--- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
+++ b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
@@@ -1,207 -1,0 +1,205 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.start.classloader.vfs;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileSystemManager;
+
+public class ContextManager {
+
+ // there is a lock per context so that one context can initialize w/o blocking another context
+ private class Context {
+ AccumuloReloadingVFSClassLoader loader;
+ ContextConfig cconfig;
+ boolean closed = false;
+
+ Context(ContextConfig cconfig) {
+ this.cconfig = cconfig;
+ }
+
+ synchronized ClassLoader getClassLoader() throws FileSystemException {
+ if (closed)
+ return null;
+
+ if (loader == null) {
+ loader = new AccumuloReloadingVFSClassLoader(cconfig.uris, vfs, parent, cconfig.preDelegation);
+ }
+
+ return loader.getClassLoader();
+ }
+
+ synchronized void close() {
+ closed = true;
+ loader.close();
+ loader = null;
+ }
+ }
+
+ private Map<String,Context> contexts = new HashMap<String,Context>();
+
+ private volatile ContextsConfig config;
+ private FileSystemManager vfs;
+ private ReloadingClassLoader parent;
+
+ ContextManager(FileSystemManager vfs, ReloadingClassLoader parent) {
+ this.vfs = vfs;
+ this.parent = parent;
+ }
+
+ public static class ContextConfig {
+ String uris;
+ boolean preDelegation;
+
+ public ContextConfig(String uris, boolean preDelegation) {
+ this.uris = uris;
+ this.preDelegation = preDelegation;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof ContextConfig) {
+ ContextConfig oc = (ContextConfig) o;
+
+ return uris.equals(oc.uris) && preDelegation == oc.preDelegation;
+ }
+
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return uris.hashCode() + (preDelegation ? Boolean.TRUE : Boolean.FALSE).hashCode();
+ }
+ }
+
+ public interface ContextsConfig {
+ ContextConfig getContextConfig(String context);
+ }
+
+ public static class DefaultContextsConfig implements ContextsConfig {
+
+ private Iterable<Entry<String,String>> config;
+
+ public DefaultContextsConfig(Iterable<Entry<String,String>> config) {
+ this.config = config;
+ }
+
+ @Override
+ public ContextConfig getContextConfig(String context) {
+
+ String key = AccumuloVFSClassLoader.VFS_CONTEXT_CLASSPATH_PROPERTY + context;
+
+ String uris = null;
+ boolean preDelegate = true;
+
+ Iterator<Entry<String,String>> iter = config.iterator();
+ while (iter.hasNext()) {
+ Entry<String,String> entry = iter.next();
+ if (entry.getKey().equals(key)) {
+ uris = entry.getValue();
+ }
+
+ if (entry.getKey().equals(key + ".delegation") && entry.getValue().trim().equalsIgnoreCase("post")) {
+ preDelegate = false;
+ }
+ }
+
+ if (uris != null)
+ return new ContextConfig(uris, preDelegate);
+
+ return null;
+ }
+ }
+
+ /**
+ * configuration must be injected for ContextManager to work
- *
- * @param config
+ */
+ public synchronized void setContextConfig(ContextsConfig config) {
+ if (this.config != null)
+ throw new IllegalStateException("Context manager config already set");
+ this.config = config;
+ }
+
+ public ClassLoader getClassLoader(String contextName) throws FileSystemException {
+
+ ContextConfig cconfig = config.getContextConfig(contextName);
+
+ if (cconfig == null)
+ throw new IllegalArgumentException("Unknown context " + contextName);
+
+ Context context = null;
+ Context contextToClose = null;
+
+ synchronized (this) {
+ // only manipulate internal data structs in this sync block... avoid creating or closing classloader, reading config, etc... basically avoid operations
+ // that may block
+ context = contexts.get(contextName);
+
+ if (context == null) {
+ context = new Context(cconfig);
+ contexts.put(contextName, context);
+ } else if (!context.cconfig.equals(cconfig)) {
+ contextToClose = context;
+ context = new Context(cconfig);
+ contexts.put(contextName, context);
+ }
+ }
+
+ if (contextToClose != null)
+ contextToClose.close();
+
+ ClassLoader loader = context.getClassLoader();
+ if (loader == null) {
+ // ooppss, context was closed by another thread, try again
+ return getClassLoader(contextName);
+ }
+
+ return loader;
+
+ }
+
+ public <U> Class<? extends U> loadClass(String context, String classname, Class<U> extension) throws ClassNotFoundException {
+ try {
+ return getClassLoader(context).loadClass(classname).asSubclass(extension);
+ } catch (IOException e) {
+ throw new ClassNotFoundException("IO Error loading class " + classname, e);
+ }
+ }
+
+ public void removeUnusedContexts(Set<String> inUse) {
+
+ Map<String,Context> unused;
+
+ synchronized (this) {
+ unused = new HashMap<String,Context>(contexts);
+ unused.keySet().removeAll(inUse);
+ contexts.keySet().removeAll(unused.keySet());
+ }
+
+ for (Context context : unused.values()) {
+ // close outside of lock
+ context.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
----------------------------------------------------------------------
diff --cc start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
index 0a6931f,0000000..277c741
mode 100644,000000..100644
--- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
+++ b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
@@@ -1,52 -1,0 +1,47 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.start.classloader.vfs;
+
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileSystemManager;
+import org.apache.commons.vfs2.impl.VFSClassLoader;
+
+/**
+ *
+ */
+public class PostDelegatingVFSClassLoader extends VFSClassLoader {
+
- /**
- * @param files
- * @param manager
- * @param parent
- * @throws FileSystemException
- */
+ public PostDelegatingVFSClassLoader(FileObject[] files, FileSystemManager manager, ClassLoader parent) throws FileSystemException {
+ super(files, manager, parent);
+ }
+
++ @Override
+ protected synchronized Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
+ Class<?> c = findLoadedClass(name);
+ if (c == null) {
+ try {
+ // try finding this class here instead of parent
+ findClass(name);
+ } catch (ClassNotFoundException e) {
+
+ }
+ }
+ return super.loadClass(name, resolve);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java
----------------------------------------------------------------------
diff --cc start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java
index 92b2720,0000000..104ea09
mode 100644,000000..100644
--- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java
+++ b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java
@@@ -1,164 -1,0 +1,159 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.start.classloader.vfs.providers;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.vfs2.CacheStrategy;
+import org.apache.commons.vfs2.Capability;
+import org.apache.commons.vfs2.FileName;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileSystemOptions;
+import org.apache.commons.vfs2.provider.AbstractFileName;
+import org.apache.commons.vfs2.provider.AbstractFileSystem;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * A VFS FileSystem that interacts with HDFS.
+ *
+ * @since 2.1
+ */
+public class HdfsFileSystem extends AbstractFileSystem
+{
+ private static final Log log = LogFactory.getLog(HdfsFileSystem.class);
+
+ private FileSystem fs;
+
- /**
- *
- * @param rootName
- * @param fileSystemOptions
- */
+ protected HdfsFileSystem(final FileName rootName, final FileSystemOptions fileSystemOptions)
+ {
+ super(rootName, null, fileSystemOptions);
+ }
+
+ /**
+ * @see org.apache.commons.vfs2.provider.AbstractFileSystem#addCapabilities(java.util.Collection)
+ */
+ @Override
+ protected void addCapabilities(final Collection<Capability> capabilities)
+ {
+ capabilities.addAll(HdfsFileProvider.CAPABILITIES);
+ }
+
+ /**
+ * @see org.apache.commons.vfs2.provider.AbstractFileSystem#close()
+ */
+ @Override
+ public void close()
+ {
+ try
+ {
+ if (null != fs)
+ {
+ fs.close();
+ }
+ }
+ catch (final IOException e)
+ {
+ throw new RuntimeException("Error closing HDFS client", e);
+ }
+ super.close();
+ }
+
+ /**
+ * @see org.apache.commons.vfs2.provider.AbstractFileSystem#createFile(org.apache.commons.vfs2.provider.AbstractFileName)
+ */
+ @Override
+ protected FileObject createFile(final AbstractFileName name) throws Exception
+ {
+ throw new FileSystemException("Operation not supported");
+ }
+
+ /**
+ * @see org.apache.commons.vfs2.provider.AbstractFileSystem#resolveFile(org.apache.commons.vfs2.FileName)
+ */
+ @Override
+ public FileObject resolveFile(final FileName name) throws FileSystemException
+ {
+
+ synchronized (this)
+ {
+ if (null == this.fs)
+ {
+ final String hdfsUri = name.getRootURI();
+ final Configuration conf = new Configuration(true);
+ conf.set(org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY, hdfsUri);
+ this.fs = null;
+ try
+ {
+ fs = org.apache.hadoop.fs.FileSystem.get(conf);
+ }
+ catch (final IOException e)
+ {
+ log.error("Error connecting to filesystem " + hdfsUri, e);
+ throw new FileSystemException("Error connecting to filesystem " + hdfsUri, e);
+ }
+ }
+ }
+
+ boolean useCache = (null != getContext().getFileSystemManager().getFilesCache());
+ FileObject file;
+ if (useCache)
+ {
+ file = this.getFileFromCache(name);
+ }
+ else
+ {
+ file = null;
+ }
+ if (null == file)
+ {
+ String path = null;
+ try
+ {
+ path = URLDecoder.decode(name.getPath(), "UTF-8");
+ }
+ catch (final UnsupportedEncodingException e)
+ {
+ path = name.getPath();
+ }
+ final Path filePath = new Path(path);
+ file = new HdfsFileObject((AbstractFileName) name, this, fs, filePath);
+ if (useCache)
+ {
+ this.putFileToCache(file);
+ }
+
+ }
+
+ /**
+ * resync the file information if requested
+ */
+ if (getFileSystemManager().getCacheStrategy().equals(CacheStrategy.ON_RESOLVE)) {
+ file.refresh();
+ }
+
+ return file;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/GetMasterStats.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/GetMasterStats.java
index 65cf80c,0000000..7b1313a
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/GetMasterStats.java
+++ b/test/src/main/java/org/apache/accumulo/test/GetMasterStats.java
@@@ -1,129 -1,0 +1,120 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
- import java.io.IOException;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.accumulo.core.client.impl.MasterClient;
- import org.apache.accumulo.core.master.MasterNotRunningException;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
+import org.apache.accumulo.core.master.thrift.RecoveryStatus;
+import org.apache.accumulo.core.master.thrift.TableInfo;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.monitor.Monitor;
+import org.apache.accumulo.server.security.SecurityConstants;
- import org.apache.thrift.transport.TTransportException;
+
+public class GetMasterStats {
- /**
- * @param args
- * @throws MasterNotRunningException
- * @throws IOException
- * @throws TTransportException
- */
+ public static void main(String[] args) throws Exception {
+ MasterClientService.Iface client = null;
+ MasterMonitorInfo stats = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(HdfsZooInstance.getInstance());
+ stats = client.getMasterStats(Tracer.traceInfo(), SecurityConstants.getSystemCredentials());
+ } finally {
+ if (client != null)
+ MasterClient.close(client);
+ }
+ out(0, "State: " + stats.state.name());
+ out(0, "Goal State: " + stats.goalState.name());
+ if (stats.serversShuttingDown != null && stats.serversShuttingDown.size() > 0) {
+ out(0, "Servers to shutdown");
+ for (String server : stats.serversShuttingDown) {
+ out(1, "%s", server);
+ }
+ }
+ out(0, "Unassigned tablets: %d", stats.unassignedTablets);
+ if (stats.badTServers != null && stats.badTServers.size() > 0) {
+ out(0, "Bad servers");
+
+ for (Entry<String,Byte> entry : stats.badTServers.entrySet()) {
+ out(1, "%s: %d", entry.getKey(), (int) entry.getValue());
+ }
+ }
+ if (stats.tableMap != null && stats.tableMap.size() > 0) {
+ out(0, "Tables");
+ for (Entry<String,TableInfo> entry : stats.tableMap.entrySet()) {
+ TableInfo v = entry.getValue();
+ out(1, "%s", entry.getKey());
+ out(2, "Records: %d", v.recs);
+ out(2, "Records in Memory: %d", v.recsInMemory);
+ out(2, "Tablets: %d", v.tablets);
+ out(2, "Online Tablets: %d", v.onlineTablets);
+ out(2, "Ingest Rate: %.2f", v.ingestRate);
+ out(2, "Query Rate: %.2f", v.queryRate);
+ }
+ }
+ if (stats.tServerInfo != null && stats.tServerInfo.size() > 0) {
+ out(0, "Tablet Servers");
+ long now = System.currentTimeMillis();
+ for (TabletServerStatus server : stats.tServerInfo) {
+ TableInfo summary = Monitor.summarizeTableStats(server);
+ out(1, "Name: %s", server.name);
+ out(2, "Ingest: %.2f", summary.ingestRate);
+ out(2, "Last Contact: %s", server.lastContact);
+ out(2, "OS Load Average: %.2f", server.osLoad);
+ out(2, "Queries: %.2f", summary.queryRate);
+ out(2, "Time Difference: %.1f", ((now - server.lastContact) / 1000.));
+ out(2, "Total Records: %d", summary.recs);
+ out(2, "Lookups: %d", server.lookups);
+ if (server.holdTime > 0)
+ out(2, "Hold Time: %d", server.holdTime);
+ if (server.tableMap != null && server.tableMap.size() > 0) {
+ out(2, "Tables");
+ for (Entry<String,TableInfo> status : server.tableMap.entrySet()) {
+ TableInfo info = status.getValue();
+ out(3, "Table: %s", status.getKey());
+ out(4, "Tablets: %d", info.onlineTablets);
+ out(4, "Records: %d", info.recs);
+ out(4, "Records in Memory: %d", info.recsInMemory);
+ out(4, "Ingest: %.2f", info.ingestRate);
+ out(4, "Queries: %.2f", info.queryRate);
+ out(4, "Major Compacting: %d", info.majors == null ? 0 : info.majors.running);
+ out(4, "Queued for Major Compaction: %d", info.majors == null ? 0 : info.majors.queued);
+ out(4, "Minor Compacting: %d", info.minors == null ? 0 : info.minors.running);
+ out(4, "Queued for Minor Compaction: %d", info.minors == null ? 0 : info.minors.queued);
+ }
+ }
+ out(2, "Recoveries: %d", server.logSorts.size());
+ for (RecoveryStatus sort : server.logSorts) {
+ out(3, "File: %s", sort.name);
+ out(3, "Progress: %.2f%%", sort.progress * 100);
+ out(3, "Time running: %s", sort.runtime / 1000.);
+ }
+ }
+ }
+ }
+
+ private static void out(int indent, String string, Object... args) {
+ for (int i = 0; i < indent; i++) {
+ System.out.print(" ");
+ }
+ System.out.println(String.format(string, args));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
index 73b73f4,0000000..16e7a98
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
+++ b/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
@@@ -1,198 -1,0 +1,195 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.FastFormat;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.tabletserver.NativeMap;
+import org.apache.hadoop.io.Text;
+
+public class NativeMapPerformanceTest {
+
+ private static final byte ROW_PREFIX[] = new byte[] {'r'};
+ private static final byte COL_PREFIX[] = new byte[] {'c'};
+
+ static Key nk(int r, int c) {
+ return new Key(new Text(FastFormat.toZeroPaddedString(r, 9, 10, ROW_PREFIX)), new Text(FastFormat.toZeroPaddedString(c, 6, 10, COL_PREFIX)));
+ }
+
+ static Mutation nm(int r) {
+ return new Mutation(new Text(FastFormat.toZeroPaddedString(r, 9, 10, ROW_PREFIX)));
+ }
+
+ static Text ET = new Text();
+
+ private static void pc(Mutation m, int c, Value v) {
+ m.put(new Text(FastFormat.toZeroPaddedString(c, 6, 10, COL_PREFIX)), ET, Long.MAX_VALUE, v);
+ }
+
+ static void runPerformanceTest(int numRows, int numCols, int numLookups, String mapType) {
+
+ SortedMap<Key,Value> tm = null;
+ NativeMap nm = null;
+
+ if (mapType.equals("SKIP_LIST"))
+ tm = new ConcurrentSkipListMap<Key,Value>();
+ else if (mapType.equals("TREE_MAP"))
+ tm = Collections.synchronizedSortedMap(new TreeMap<Key,Value>());
+ else if (mapType.equals("NATIVE_MAP"))
+ nm = new NativeMap();
+ else
+ throw new IllegalArgumentException(" map type must be SKIP_LIST, TREE_MAP, or NATIVE_MAP");
+
+ Random rand = new Random(19);
+
+ // puts
+ long tps = System.currentTimeMillis();
+
+ if (nm != null) {
+ for (int i = 0; i < numRows; i++) {
+ int row = rand.nextInt(1000000000);
+ Mutation m = nm(row);
+ for (int j = 0; j < numCols; j++) {
+ int col = rand.nextInt(1000000);
+ Value val = new Value("test".getBytes(Constants.UTF8));
+ pc(m, col, val);
+ }
+ nm.mutate(m, i);
+ }
+ } else {
+ for (int i = 0; i < numRows; i++) {
+ int row = rand.nextInt(1000000000);
+ for (int j = 0; j < numCols; j++) {
+ int col = rand.nextInt(1000000);
+ Key key = nk(row, col);
+ Value val = new Value("test".getBytes(Constants.UTF8));
+ tm.put(key, val);
+ }
+ }
+ }
+
+ long tpe = System.currentTimeMillis();
+
+ // Iteration
+ Iterator<Entry<Key,Value>> iter;
+ if (nm != null) {
+ iter = nm.iterator();
+ } else {
+ iter = tm.entrySet().iterator();
+ }
+
+ long tis = System.currentTimeMillis();
+
+ while (iter.hasNext()) {
+ iter.next();
+ }
+
+ long tie = System.currentTimeMillis();
+
+ rand = new Random(19);
+ int rowsToLookup[] = new int[numLookups];
+ int colsToLookup[] = new int[numLookups];
+ for (int i = 0; i < Math.min(numLookups, numRows); i++) {
+ int row = rand.nextInt(1000000000);
+ int col = -1;
+ for (int j = 0; j < numCols; j++) {
+ col = rand.nextInt(1000000);
+ }
+
+ rowsToLookup[i] = row;
+ colsToLookup[i] = col;
+ }
+
+ // get
+
+ long tgs = System.currentTimeMillis();
+ if (nm != null) {
+ for (int i = 0; i < numLookups; i++) {
+ Key key = nk(rowsToLookup[i], colsToLookup[i]);
+ if (nm.get(key) == null) {
+ throw new RuntimeException("Did not find " + rowsToLookup[i] + " " + colsToLookup[i] + " " + i);
+ }
+ }
+ } else {
+ for (int i = 0; i < numLookups; i++) {
+ Key key = nk(rowsToLookup[i], colsToLookup[i]);
+ if (tm.get(key) == null) {
+ throw new RuntimeException("Did not find " + rowsToLookup[i] + " " + colsToLookup[i] + " " + i);
+ }
+ }
+ }
+ long tge = System.currentTimeMillis();
+
+ long memUsed = 0;
+ if (nm != null) {
+ memUsed = nm.getMemoryUsed();
+ }
+
+ int size = (nm == null ? tm.size() : nm.size());
+
+ // delete
+ long tds = System.currentTimeMillis();
+
+ if (nm != null)
+ nm.delete();
+
+ long tde = System.currentTimeMillis();
+
+ if (tm != null)
+ tm.clear();
+
+ System.gc();
+ System.gc();
+ System.gc();
+ System.gc();
+
+ UtilWaitThread.sleep(3000);
+
+ System.out.printf("mapType:%10s put rate:%,6.2f scan rate:%,6.2f get rate:%,6.2f delete time : %6.2f mem : %,d%n", "" + mapType, (numRows * numCols)
+ / ((tpe - tps) / 1000.0), (size) / ((tie - tis) / 1000.0), numLookups / ((tge - tgs) / 1000.0), (tde - tds) / 1000.0, memUsed);
+
+ }
+
- /**
- * @param args
- */
+ public static void main(String[] args) {
+
+ if (args.length != 3) {
+ throw new IllegalArgumentException("Usage : " + NativeMapPerformanceTest.class.getName() + " <map type> <rows> <columns>");
+ }
+
+ String mapType = args[0];
+ int rows = Integer.parseInt(args[1]);
+ int cols = Integer.parseInt(args[2]);
+
+ runPerformanceTest(rows, cols, 10000, mapType);
+ runPerformanceTest(rows, cols, 10000, mapType);
+ runPerformanceTest(rows, cols, 10000, mapType);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
index 8411c86,0000000..5115541
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
+++ b/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
@@@ -1,277 -1,0 +1,274 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.OpTimer;
+import org.apache.accumulo.server.tabletserver.NativeMap;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+public class NativeMapStressTest {
+
+ private static final Logger log = Logger.getLogger(NativeMapStressTest.class);
+
- /**
- * @param args
- */
+ public static void main(String[] args) {
+ testLotsOfMapDeletes(true);
+ testLotsOfMapDeletes(false);
+ testLotsOfOverwrites();
+ testLotsOfGetsAndScans();
+ }
+
+ private static void put(NativeMap nm, String row, String val, int mc) {
+ Mutation m = new Mutation(new Text(row));
+ m.put(new Text(), new Text(), Long.MAX_VALUE, new Value(val.getBytes(Constants.UTF8)));
+ nm.mutate(m, mc);
+ }
+
+ private static void testLotsOfGetsAndScans() {
+
+ ArrayList<Thread> threads = new ArrayList<Thread>();
+
+ final int numThreads = 8;
+ final int totalGets = 100000000;
+ final int mapSizePerThread = (int) (4000000 / (double) numThreads);
+ final int getsPerThread = (int) (totalGets / (double) numThreads);
+
+ for (int tCount = 0; tCount < numThreads; tCount++) {
+ Runnable r = new Runnable() {
+ @Override
+ public void run() {
+ NativeMap nm = new NativeMap();
+
+ Random r = new Random();
+
+ OpTimer opTimer = new OpTimer(log, Level.INFO);
+
+ opTimer.start("Creating map of size " + mapSizePerThread);
+
+ for (int i = 0; i < mapSizePerThread; i++) {
+ String row = String.format("r%08d", i);
+ String val = row + "v";
+ put(nm, row, val, i);
+ }
+
+ opTimer.stop("Created map of size " + nm.size() + " in %DURATION%");
+
+ opTimer.start("Doing " + getsPerThread + " gets()");
+
+ for (int i = 0; i < getsPerThread; i++) {
+ String row = String.format("r%08d", r.nextInt(mapSizePerThread));
+ String val = row + "v";
+
+ Value value = nm.get(new Key(new Text(row)));
+ if (value == null || !value.toString().equals(val)) {
+ log.error("nm.get(" + row + ") failed");
+ }
+ }
+
+ opTimer.stop("Finished " + getsPerThread + " gets in %DURATION%");
+
+ int scanned = 0;
+
+ opTimer.start("Doing " + getsPerThread + " random iterations");
+
+ for (int i = 0; i < getsPerThread; i++) {
+ int startRow = r.nextInt(mapSizePerThread);
+ String row = String.format("r%08d", startRow);
+
+ Iterator<Entry<Key,Value>> iter = nm.iterator(new Key(new Text(row)));
+
+ int count = 0;
+
+ while (iter.hasNext() && count < 10) {
+ String row2 = String.format("r%08d", startRow + count);
+ String val2 = row2 + "v";
+
+ Entry<Key,Value> entry = iter.next();
+ if (!entry.getValue().toString().equals(val2) || !entry.getKey().equals(new Key(new Text(row2)))) {
+ log.error("nm.iter(" + row2 + ") failed row = " + row + " count = " + count + " row2 = " + row + " val2 = " + val2);
+ }
+
+ count++;
+ }
+
+ scanned += count;
+ }
+
+ opTimer.stop("Finished " + getsPerThread + " random iterations (scanned = " + scanned + ") in %DURATION%");
+
+ nm.delete();
+ }
+ };
+
+ Thread t = new Thread(r);
+ t.start();
+
+ threads.add(t);
+ }
+
+ for (Thread thread : threads) {
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private static void testLotsOfMapDeletes(final boolean doRemoves) {
+ final int numThreads = 8;
+ final int rowRange = 10000;
+ final int mapsPerThread = 50;
+ final int totalInserts = 100000000;
+ final int insertsPerMapPerThread = (int) (totalInserts / (double) numThreads / mapsPerThread);
+
+ System.out.println("insertsPerMapPerThread " + insertsPerMapPerThread);
+
+ ArrayList<Thread> threads = new ArrayList<Thread>();
+
+ for (int i = 0; i < numThreads; i++) {
+ Runnable r = new Runnable() {
+ @Override
+ public void run() {
+
+ int inserts = 0;
+ int removes = 0;
+
+ for (int i = 0; i < mapsPerThread; i++) {
+
+ NativeMap nm = new NativeMap();
+
+ for (int j = 0; j < insertsPerMapPerThread; j++) {
+ String row = String.format("r%08d", j % rowRange);
+ String val = row + "v";
+ put(nm, row, val, j);
+ inserts++;
+ }
+
+ if (doRemoves) {
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+ while (iter.hasNext()) {
+ iter.next();
+ iter.remove();
+ removes++;
+ }
+ }
+
+ nm.delete();
+ }
+
+ System.out.println("inserts " + inserts + " removes " + removes + " " + Thread.currentThread().getName());
+ }
+ };
+
+ Thread t = new Thread(r);
+ t.start();
+
+ threads.add(t);
+ }
+
+ for (Thread thread : threads) {
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private static void testLotsOfOverwrites() {
+ final Map<Integer,NativeMap> nativeMaps = new HashMap<Integer,NativeMap>();
+
+ int numThreads = 8;
+ final int insertsPerThread = (int) (100000000 / (double) numThreads);
+ final int rowRange = 10000;
+ final int numMaps = 50;
+
+ ArrayList<Thread> threads = new ArrayList<Thread>();
+
+ for (int i = 0; i < numThreads; i++) {
+ Runnable r = new Runnable() {
+ @Override
+ public void run() {
+ Random r = new Random();
+ int inserts = 0;
+
+ for (int i = 0; i < insertsPerThread / 100.0; i++) {
+ int map = r.nextInt(numMaps);
+
+ NativeMap nm;
+
+ synchronized (nativeMaps) {
+ nm = nativeMaps.get(map);
+ if (nm == null) {
+ nm = new NativeMap();
+ nativeMaps.put(map, nm);
+
+ }
+ }
+
+ synchronized (nm) {
+ for (int j = 0; j < 100; j++) {
+ String row = String.format("r%08d", r.nextInt(rowRange));
+ String val = row + "v";
+ put(nm, row, val, j);
+ inserts++;
+ }
+ }
+ }
+
+ System.out.println("inserts " + inserts + " " + Thread.currentThread().getName());
+ }
+ };
+
+ Thread t = new Thread(r);
+ t.start();
+
+ threads.add(t);
+ }
+
+ for (Thread thread : threads) {
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ }
+ }
+
+ Set<Entry<Integer,NativeMap>> es = nativeMaps.entrySet();
+ for (Entry<Integer,NativeMap> entry : es) {
+ entry.getValue().delete();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
index a35ca66,0000000..0d52f12
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
@@@ -1,181 -1,0 +1,180 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.continuous;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
+import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.server.util.reflection.CounterUtils;
+import org.apache.accumulo.test.continuous.ContinuousIngest.BaseOpts;
+import org.apache.accumulo.test.continuous.ContinuousIngest.ShortConverter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.validators.PositiveInteger;
+
+/**
+ * A map only job that reads a table created by continuous ingest and creates doubly linked list. This map reduce job tests the ability of a map only job to
+ * read and write to accumulo at the same time. This map reduce job mutates the table in such a way that it should not create any undefined nodes.
+ *
+ */
+public class ContinuousMoru extends Configured implements Tool {
+ private static final String PREFIX = ContinuousMoru.class.getSimpleName() + ".";
+ private static final String MAX_CQ = PREFIX + "MAX_CQ";
+ private static final String MAX_CF = PREFIX + "MAX_CF";
+ private static final String MAX = PREFIX + "MAX";
+ private static final String MIN = PREFIX + "MIN";
+ private static final String CI_ID = PREFIX + "CI_ID";
+
+ static enum Counts {
+ SELF_READ;
+ }
+
+ public static class CMapper extends Mapper<Key,Value,Text,Mutation> {
+
+ private short max_cf;
+ private short max_cq;
+ private Random random;
+ private String ingestInstanceId;
+ private byte[] iiId;
+ private long count;
+
+ private static final ColumnVisibility EMPTY_VIS = new ColumnVisibility();
+
+ @Override
+ public void setup(Context context) throws IOException, InterruptedException {
+ int max_cf = context.getConfiguration().getInt(MAX_CF, -1);
+ int max_cq = context.getConfiguration().getInt(MAX_CQ, -1);
+
+ if (max_cf > Short.MAX_VALUE || max_cq > Short.MAX_VALUE)
+ throw new IllegalArgumentException();
+
+ this.max_cf = (short) max_cf;
+ this.max_cq = (short) max_cq;
+
+ random = new Random();
+ ingestInstanceId = context.getConfiguration().get(CI_ID);
+ iiId = ingestInstanceId.getBytes(Constants.UTF8);
+
+ count = 0;
+ }
+
+ @Override
+ public void map(Key key, Value data, Context context) throws IOException, InterruptedException {
+
+ ContinuousWalk.validate(key, data);
+
+ if (WritableComparator.compareBytes(iiId, 0, iiId.length, data.get(), 0, iiId.length) != 0) {
+ // only rewrite data not written by this M/R job
+ byte[] val = data.get();
+
+ int offset = ContinuousWalk.getPrevRowOffset(val);
+ if (offset > 0) {
+ long rowLong = Long.parseLong(new String(val, offset, 16, Constants.UTF8), 16);
+ Mutation m = ContinuousIngest.genMutation(rowLong, random.nextInt(max_cf), random.nextInt(max_cq), EMPTY_VIS, iiId, count++, key.getRowData()
+ .toArray(), random, true);
+ context.write(null, m);
+ }
+
+ } else {
+ CounterUtils.increment(context.getCounter(Counts.SELF_READ));
+ }
+ }
+ }
+
+ static class Opts extends BaseOpts {
+ @Parameter(names = "--maxColF", description = "maximum column family value to use", converter=ShortConverter.class)
+ short maxColF = Short.MAX_VALUE;
+
+ @Parameter(names = "--maxColQ", description = "maximum column qualifier value to use", converter=ShortConverter.class)
+ short maxColQ = Short.MAX_VALUE;
+
+ @Parameter(names = "--maxMappers", description = "the maximum number of mappers to use", required = true, validateWith = PositiveInteger.class)
+ int maxMaps = 0;
+ }
+
+ @Override
+ public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException, AccumuloSecurityException {
+ Opts opts = new Opts();
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(ContinuousMoru.class.getName(), args, bwOpts);
+
+ Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+ job.setJarByClass(this.getClass());
+
+ job.setInputFormatClass(AccumuloInputFormat.class);
+ opts.setAccumuloConfigs(job);
+
+ // set up ranges
+ try {
+ Set<Range> ranges = opts.getConnector().tableOperations().splitRangeByTablets(opts.getTableName(), new Range(), opts.maxMaps);
+ AccumuloInputFormat.setRanges(job, ranges);
+ AccumuloInputFormat.setAutoAdjustRanges(job, false);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ job.setMapperClass(CMapper.class);
+
+ job.setNumReduceTasks(0);
+
+ job.setOutputFormatClass(AccumuloOutputFormat.class);
+ AccumuloOutputFormat.setBatchWriterOptions(job, bwOpts.getBatchWriterConfig());
+
+ Configuration conf = job.getConfiguration();
+ conf.setLong(MIN, opts.min);
+ conf.setLong(MAX, opts.max);
+ conf.setInt(MAX_CF, opts.maxColF);
+ conf.setInt(MAX_CQ, opts.maxColQ);
+ conf.set(CI_ID, UUID.randomUUID().toString());
+
+ job.waitForCompletion(true);
+ opts.stopTracing();
+ return job.isSuccessful() ? 0 : 1;
+ }
+
+ /**
+ *
+ * @param args
+ * instanceName zookeepers username password table columns outputpath
- * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(CachedConfiguration.getInstance(), new ContinuousMoru(), args);
+ if (res != 0)
+ System.exit(res);
+ }
+}