You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2013/12/03 22:16:20 UTC
[1/5] ACCUMULO-1961 Fix warnings
Updated Branches:
refs/heads/1.4.5-SNAPSHOT 47403ba4f -> a3a889554
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/server/src/main/java/org/apache/accumulo/server/test/CreateRandomMapFile.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/CreateRandomMapFile.java b/src/server/src/main/java/org/apache/accumulo/server/test/CreateRandomMapFile.java
index 2472107..6f63147 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/test/CreateRandomMapFile.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/test/CreateRandomMapFile.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.io.Text;
/**
* @deprecated since 1.4 (will become CreateRandomRFile)
*/
+@Deprecated
public class CreateRandomMapFile {
private static int num;
private static String file;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/server/src/main/java/org/apache/accumulo/server/test/DumpMapFile.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/DumpMapFile.java b/src/server/src/main/java/org/apache/accumulo/server/test/DumpMapFile.java
index f29ca76..91439dd 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/test/DumpMapFile.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/test/DumpMapFile.java
@@ -29,6 +29,7 @@ import org.apache.log4j.Logger;
/**
* @deprecated since 1.4
*/
+@Deprecated
public class DumpMapFile {
private static final Logger log = Logger.getLogger(DumpMapFile.class);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/server/src/main/java/org/apache/accumulo/server/test/MMFITest.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/MMFITest.java b/src/server/src/main/java/org/apache/accumulo/server/test/MMFITest.java
index 14fbe26..4bc94e3 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/test/MMFITest.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/test/MMFITest.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.io.Text;
/**
* @deprecated since 1.4
*/
+@Deprecated
public class MMFITest {
private static MyMapFile.Reader[] mapfiles = null;
private static Text startRow;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/server/src/main/java/org/apache/accumulo/server/test/MidPointPerfTest2.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/MidPointPerfTest2.java b/src/server/src/main/java/org/apache/accumulo/server/test/MidPointPerfTest2.java
index 52094d0..2029b82 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/test/MidPointPerfTest2.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/test/MidPointPerfTest2.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.io.WritableComparable;
/**
* @deprecated since 1.4
*/
+@Deprecated
class MultipleIndexIterator2 {
private MySequenceFile.Reader[] readers;
@@ -125,6 +126,7 @@ class MultipleIndexIterator2 {
/**
* @deprecated since 1.4
*/
+@Deprecated
public class MidPointPerfTest2 {
private static String dir;
private static int numFiles;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/server/src/main/java/org/apache/accumulo/server/util/CountDiskRows.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/util/CountDiskRows.java b/src/server/src/main/java/org/apache/accumulo/server/util/CountDiskRows.java
index 3b7ea1d..f837721 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/util/CountDiskRows.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/util/CountDiskRows.java
@@ -33,6 +33,7 @@ import org.apache.log4j.Logger;
/**
* @deprecated since 1.4
*/
+@Deprecated
public class CountDiskRows {
private static final Logger log = Logger.getLogger(CountDiskRows.class);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/server/src/main/java/org/apache/accumulo/server/util/DumpMapFile.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/util/DumpMapFile.java b/src/server/src/main/java/org/apache/accumulo/server/util/DumpMapFile.java
index 36c5b7e..dfcc6da 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/util/DumpMapFile.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/util/DumpMapFile.java
@@ -29,6 +29,7 @@ import org.apache.log4j.Logger;
/**
* @deprecated
*/
+@Deprecated
public class DumpMapFile {
private static final Logger log = Logger.getLogger(DumpMapFile.class);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/server/src/main/java/org/apache/accumulo/server/util/DumpTable.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/util/DumpTable.java b/src/server/src/main/java/org/apache/accumulo/server/util/DumpTable.java
index 08836da..d642fc9 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/util/DumpTable.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/util/DumpTable.java
@@ -32,6 +32,7 @@ import org.apache.log4j.Logger;
/**
* @deprecated since 1.4
*/
+@Deprecated
public class DumpTable {
private static final Logger log = Logger.getLogger(DumpTable.class);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/server/src/main/java/org/apache/accumulo/server/util/MapFilePerformanceTest.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/util/MapFilePerformanceTest.java b/src/server/src/main/java/org/apache/accumulo/server/util/MapFilePerformanceTest.java
index 1e5170d..2efaa97 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/util/MapFilePerformanceTest.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/util/MapFilePerformanceTest.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.io.Text;
/**
* @deprecated since 1.4
*/
+@Deprecated
public class MapFilePerformanceTest {
public static String[] createMapFiles(String input, String output, int blocksize, int mapFiles) throws IOException {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/server/src/main/java/org/apache/accumulo/server/util/OfflineMetadataScanner.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/util/OfflineMetadataScanner.java b/src/server/src/main/java/org/apache/accumulo/server/util/OfflineMetadataScanner.java
index 2487195..2584323 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/util/OfflineMetadataScanner.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/util/OfflineMetadataScanner.java
@@ -56,10 +56,10 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class OfflineMetadataScanner extends ScannerOptions implements Scanner {
-
+
private Set<String> allFiles = new HashSet<String>();
private Range range = new Range();
-
+
private List<SortedKeyValueIterator<Key,Value>> openMapFiles(Collection<String> files, FileSystem fs, Configuration conf) throws IOException {
List<SortedKeyValueIterator<Key,Value>> readers = new ArrayList<SortedKeyValueIterator<Key,Value>>();
for (String file : files) {
@@ -68,7 +68,7 @@ public class OfflineMetadataScanner extends ScannerOptions implements Scanner {
}
return readers;
}
-
+
private SortedKeyValueIterator<Key,Value> createSystemIter(Range r, List<SortedKeyValueIterator<Key,Value>> readers, HashSet<Column> columns)
throws IOException {
MultiIterator multiIterator = new MultiIterator(readers, false);
@@ -76,77 +76,77 @@ public class OfflineMetadataScanner extends ScannerOptions implements Scanner {
ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(delIter);
ColumnQualifierFilter colFilter = new ColumnQualifierFilter(cfsi, columns);
VisibilityFilter visFilter = new VisibilityFilter(colFilter, Constants.NO_AUTHS, new byte[0]);
-
+
visFilter.seek(r, LocalityGroupUtil.EMPTY_CF_SET, false);
-
+
VersioningIterator vi = new VersioningIterator();
Map<String,String> opts = new HashMap<String,String>();
opts.put("maxVersions", "1");
vi.init(visFilter, opts, null);
-
+
return vi;
}
-
+
private static class MyEntry implements Map.Entry<Key,Value> {
-
+
private Key k;
private Value v;
-
+
MyEntry(Key k, Value v) {
this.k = k;
this.v = v;
}
-
+
@Override
public Key getKey() {
return k;
}
-
+
@Override
public Value getValue() {
return v;
}
-
+
@Override
public Value setValue(Value value) {
throw new UnsupportedOperationException();
}
-
+
}
-
+
public OfflineMetadataScanner() throws IOException {
super();
-
+
List<LogEntry> rwal;
try {
rwal = MetadataTable.getLogEntries(null, Constants.ROOT_TABLET_EXTENT);
} catch (Exception e) {
throw new RuntimeException("Failed to check if root tablet has write ahead log entries", e);
}
-
+
if (rwal.size() > 0) {
throw new RuntimeException("Root tablet has write ahead logs, can not scan offline");
}
-
+
Configuration conf = CachedConfiguration.getInstance();
FileSystem fs = FileSystem.get(conf);
-
+
FileStatus[] rootFiles = fs.listStatus(new Path(ServerConstants.getRootTabletDir()));
-
+
for (FileStatus rootFile : rootFiles) {
allFiles.add(rootFile.getPath().toString());
}
-
+
List<SortedKeyValueIterator<Key,Value>> readers = openMapFiles(allFiles, fs, conf);
-
+
HashSet<Column> columns = new HashSet<Column>();
columns.add(new Column(TextUtil.getBytes(Constants.METADATA_DATAFILE_COLUMN_FAMILY), null, null));
columns.add(new Column(TextUtil.getBytes(Constants.METADATA_LOG_COLUMN_FAMILY), null, null));
-
+
SortedKeyValueIterator<Key,Value> ssi = createSystemIter(new Range(), readers, columns);
-
+
int walogs = 0;
-
+
while (ssi.hasTop()) {
if (ssi.getTopKey().compareColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY) == 0) {
allFiles.add(ServerConstants.getMetadataTableDir() + "/" + ssi.getTopKey().getColumnQualifier().toString());
@@ -155,38 +155,38 @@ public class OfflineMetadataScanner extends ScannerOptions implements Scanner {
}
ssi.next();
}
-
+
closeReaders(readers);
-
+
if (walogs > 0) {
throw new RuntimeException("Metadata tablets have write ahead logs, can not scan offline");
}
}
-
+
private void closeReaders(List<SortedKeyValueIterator<Key,Value>> readers) throws IOException {
for (SortedKeyValueIterator<Key,Value> reader : readers) {
((FileSKVIterator) reader).close();
}
}
-
+
@Override
public int getBatchSize() {
throw new UnsupportedOperationException();
}
-
+
@Override
public Range getRange() {
return range;
}
-
+
@Override
public int getTimeOut() {
throw new UnsupportedOperationException();
}
-
+
@Override
public Iterator<Entry<Key,Value>> iterator() {
-
+
final SortedKeyValueIterator<Key,Value> ssi;
final List<SortedKeyValueIterator<Key,Value>> readers;
try {
@@ -197,20 +197,20 @@ public class OfflineMetadataScanner extends ScannerOptions implements Scanner {
} catch (IOException e) {
throw new RuntimeException(e);
}
-
+
return new Iterator<Entry<Key,Value>>() {
-
+
@Override
public boolean hasNext() {
return ssi.hasTop() && range.contains(ssi.getTopKey());
}
-
+
@Override
public Entry<Key,Value> next() {
if (!ssi.hasTop()) {
throw new NoSuchElementException();
}
-
+
MyEntry e = new MyEntry(new Key(ssi.getTopKey()), new Value(ssi.getTopValue()));
try {
ssi.next();
@@ -219,50 +219,51 @@ public class OfflineMetadataScanner extends ScannerOptions implements Scanner {
}
return e;
}
-
+
@Override
public void remove() {
throw new UnsupportedOperationException();
}
-
+
};
}
-
+
@Override
public void setBatchSize(int size) {
throw new UnsupportedOperationException();
}
-
+
@Override
public void setRange(Range range) {
this.range = range;
}
-
+
@Override
public void setTimeOut(int timeOut) {
throw new UnsupportedOperationException();
}
-
+
+ @Deprecated
@Override
public void setScanIterators(int priority, String iteratorClass, String iteratorName) {
throw new UnsupportedOperationException();
}
-
+
@Override
public void enableIsolation() {
// should always give an isolated view since it is scanning immutable files
}
-
+
@Override
public void disableIsolation() {
-
+
}
-
+
public static void main(String[] args) throws IOException {
OfflineMetadataScanner scanner = new OfflineMetadataScanner();
scanner.setRange(Constants.METADATA_KEYSPACE);
for (Entry<Key,Value> entry : scanner)
System.out.println(entry.getKey() + " " + entry.getValue());
}
-
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/server/src/test/java/org/apache/accumulo/server/util/InitializeTest.java
----------------------------------------------------------------------
diff --git a/src/server/src/test/java/org/apache/accumulo/server/util/InitializeTest.java b/src/server/src/test/java/org/apache/accumulo/server/util/InitializeTest.java
index 138a0ac..4a406aa 100644
--- a/src/server/src/test/java/org/apache/accumulo/server/util/InitializeTest.java
+++ b/src/server/src/test/java/org/apache/accumulo/server/util/InitializeTest.java
@@ -16,6 +16,14 @@
*/
package org.apache.accumulo.server.util;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
import java.io.IOException;
import java.net.URI;
@@ -25,12 +33,9 @@ import org.apache.accumulo.server.zookeeper.IZooReaderWriter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.easymock.Capture;
-import static org.easymock.EasyMock.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import static org.junit.Assert.*;
/**
* This test is not thread-safe.
[3/5] ACCUMULO-1961 Fix warnings
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
index 517575c..14f95a1 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
@@ -391,6 +391,7 @@ public class IntersectingIterator implements SortedKeyValueIterator<Key,Value> {
* @deprecated since 1.4. To be made protected. Do not interact with flags string directly, just use
* {@link #setColumnFamilies(IteratorSetting, Text[], boolean[])}.
*/
+ @Deprecated
public static String encodeColumns(Text[] columns) {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < columns.length; i++) {
@@ -406,6 +407,7 @@ public class IntersectingIterator implements SortedKeyValueIterator<Key,Value> {
* @deprecated since 1.4. To be made protected. Do not interact with flags string directly, just use
* {@link #setColumnFamilies(IteratorSetting, Text[], boolean[])}.
*/
+ @Deprecated
public static String encodeBooleans(boolean[] flags) {
byte[] bytes = new byte[flags.length];
for (int i = 0; i < flags.length; i++) {
@@ -434,6 +436,7 @@ public class IntersectingIterator implements SortedKeyValueIterator<Key,Value> {
* @deprecated since 1.4. To be made protected. Do not interact with flags string directly, just use
* {@link #setColumnFamilies(IteratorSetting, Text[], boolean[])}.
*/
+ @Deprecated
public static boolean[] decodeBooleans(String flags) {
// return null of there were no flags
if (flags == null)
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java b/src/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
index cfbed38..f8de400 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
@@ -117,8 +117,6 @@ import org.apache.accumulo.core.util.shell.commands.QuotedStringTokenizer;
import org.apache.accumulo.core.util.shell.commands.RenameTableCommand;
import org.apache.accumulo.core.util.shell.commands.RevokeCommand;
import org.apache.accumulo.core.util.shell.commands.ScanCommand;
-import org.apache.accumulo.core.util.shell.commands.SelectCommand;
-import org.apache.accumulo.core.util.shell.commands.SelectrowCommand;
import org.apache.accumulo.core.util.shell.commands.SetAuthsCommand;
import org.apache.accumulo.core.util.shell.commands.SetGroupsCommand;
import org.apache.accumulo.core.util.shell.commands.SetIterCommand;
@@ -153,11 +151,11 @@ import org.apache.log4j.Logger;
public class Shell {
public static final Logger log = Logger.getLogger(Shell.class);
private static final Logger audit = Logger.getLogger(Shell.class.getName() + ".audit");
-
+
public static final int NO_FIXED_ARG_LENGTH_CHECK = -1;
private static final String SHELL_DESCRIPTION = "Shell - Apache Accumulo Interactive Shell";
private static final String DEFAULT_AUTH_TIMEOUT = "60"; // in minutes
-
+
private int exitCode = 0;
private String tableName;
private Instance instance;
@@ -167,129 +165,129 @@ public class Shell {
private Class<? extends Formatter> defaultFormatterClass = DefaultFormatter.class;
private Class<? extends Formatter> binaryFormatterClass = BinaryFormatter.class;
public Map<String,List<IteratorSetting>> scanIteratorOptions = new HashMap<String,List<IteratorSetting>>();
-
+
private Token rootToken;
public final Map<String,Command> commandFactory = new TreeMap<String,Command>();
private boolean configError = false;
-
+
// Global options flags
public static final String userOption = "u";
public static final String tableOption = "t";
public static final String helpOption = "?";
public static final String helpLongOption = "help";
-
+
// exit if true
private boolean exit = false;
-
+
// file to execute commands from
private String execFile = null;
// single command to execute from the command line
private String execCommand = null;
private boolean verbose = true;
-
+
private boolean tabCompletion;
private boolean disableAuthTimeout;
private long authTimeout;
private long lastUserActivity = System.currentTimeMillis();
-
+
@SuppressWarnings("deprecation")
public void config(String... args) {
Options opts = new Options();
-
+
Option usernameOption = new Option("u", "user", true, "username (defaults to your OS user)");
usernameOption.setArgName("user");
opts.addOption(usernameOption);
-
+
Option passwOption = new Option("p", "password", true, "password (prompt for password if this option is missing)");
passwOption.setArgName("pass");
opts.addOption(passwOption);
-
+
Option tabCompleteOption = new Option(null, "disable-tab-completion", false, "disables tab completion (for less overhead when scripting)");
opts.addOption(tabCompleteOption);
-
+
Option debugOption = new Option(null, "debug", false, "enables client debugging");
opts.addOption(debugOption);
-
+
Option fakeOption = new Option(null, "fake", false, "fake a connection to accumulo");
opts.addOption(fakeOption);
-
+
Option helpOpt = new Option(helpOption, helpLongOption, false, "display this help");
opts.addOption(helpOpt);
-
+
Option execCommandOpt = new Option("e", "execute-command", true, "executes a command, and then exits");
opts.addOption(execCommandOpt);
-
+
OptionGroup execFileGroup = new OptionGroup();
-
+
Option execfileOption = new Option("f", "execute-file", true, "executes commands from a file at startup");
execfileOption.setArgName("file");
execFileGroup.addOption(execfileOption);
-
+
Option execfileVerboseOption = new Option("fv", "execute-file-verbose", true, "executes commands from a file at startup, with commands shown");
execfileVerboseOption.setArgName("file");
execFileGroup.addOption(execfileVerboseOption);
-
+
opts.addOptionGroup(execFileGroup);
-
+
OptionGroup instanceOptions = new OptionGroup();
-
+
Option hdfsZooInstance = new Option("h", "hdfsZooInstance", false, "use hdfs zoo instance");
instanceOptions.addOption(hdfsZooInstance);
-
+
Option zooKeeperInstance = new Option("z", "zooKeeperInstance", true, "use a zookeeper instance with the given instance name and list of zoo hosts");
zooKeeperInstance.setArgName("name hosts");
zooKeeperInstance.setArgs(2);
instanceOptions.addOption(zooKeeperInstance);
-
+
opts.addOptionGroup(instanceOptions);
-
+
OptionGroup authTimeoutOptions = new OptionGroup();
-
+
Option authTimeoutOpt = new Option(null, "auth-timeout", true, "minutes the shell can be idle without re-entering a password (default "
+ DEFAULT_AUTH_TIMEOUT + " min)");
authTimeoutOpt.setArgName("minutes");
authTimeoutOptions.addOption(authTimeoutOpt);
-
+
Option disableAuthTimeoutOpt = new Option(null, "disable-auth-timeout", false, "disables requiring the user to re-type a password after being idle");
authTimeoutOptions.addOption(disableAuthTimeoutOpt);
-
+
opts.addOptionGroup(authTimeoutOptions);
-
+
CommandLine cl;
try {
cl = new BasicParser().parse(opts, args);
if (cl.getArgs().length > 0)
throw new ParseException("Unrecognized arguments: " + cl.getArgList());
-
+
if (cl.hasOption(helpOpt.getOpt())) {
configError = true;
printHelp("shell", SHELL_DESCRIPTION, opts);
return;
}
-
+
setDebugging(cl.hasOption(debugOption.getLongOpt()));
authTimeout = Integer.parseInt(cl.getOptionValue(authTimeoutOpt.getLongOpt(), DEFAULT_AUTH_TIMEOUT)) * 60 * 1000;
disableAuthTimeout = cl.hasOption(disableAuthTimeoutOpt.getLongOpt());
-
+
if (cl.hasOption(zooKeeperInstance.getOpt()) && cl.getOptionValues(zooKeeperInstance.getOpt()).length != 2)
throw new MissingArgumentException(zooKeeperInstance);
-
+
} catch (Exception e) {
configError = true;
printException(e);
printHelp("shell", SHELL_DESCRIPTION, opts);
return;
}
-
+
// get the options that were parsed
String sysUser = System.getProperty("user.name");
if (sysUser == null)
sysUser = "root";
String user = cl.getOptionValue(usernameOption.getOpt(), sysUser);
-
+
String passw = cl.getOptionValue(passwOption.getOpt(), null);
tabCompletion = !cl.hasOption(tabCompleteOption.getLongOpt());
-
+
// should only be one instance option set
instance = null;
if (cl.hasOption(fakeOption.getLongOpt())) {
@@ -302,14 +300,14 @@ public class Shell {
} else {
instance = getDefaultInstance(AccumuloConfiguration.getSiteConfiguration());
}
-
+
// process default parameters if unspecified
byte[] pass;
try {
if (!cl.hasOption(fakeOption.getLongOpt())) {
DistributedTrace.enable(instance, new ZooReader(instance), "shell", InetAddress.getLocalHost().getHostName());
}
-
+
this.reader = new ConsoleReader();
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
@@ -317,7 +315,7 @@ public class Shell {
reader.getTerminal().enableEcho();
}
});
-
+
if (passw == null)
passw = reader.readLine("Enter current password for '" + user + "'@'" + instance.getInstanceName() + "': ", '*');
if (passw == null) {
@@ -325,17 +323,17 @@ public class Shell {
configError = true;
return;
} // user canceled
-
+
pass = passw.getBytes();
this.setTableName("");
connector = instance.getConnector(user, pass);
this.credentials = new AuthInfo(user, ByteBuffer.wrap(pass), connector.getInstance().getInstanceID());
-
+
} catch (Exception e) {
printException(e);
configError = true;
}
-
+
// decide whether to execute commands from a file and quit
if (cl.hasOption(execfileOption.getOpt())) {
execFile = cl.getOptionValue(execfileOption.getOpt());
@@ -347,7 +345,7 @@ public class Shell {
execCommand = cl.getOptionValue(execCommandOpt.getOpt());
verbose = false;
}
-
+
rootToken = new Token();
Command external[] = {new AboutCommand(), new AddSplitsCommand(), new AuthenticateCommand(), new ByeCommand(), new ClasspathCommand(), new ClearCommand(),
new CloneTableCommand(), new ClsCommand(), new CompactCommand(), new ConfigCommand(), new CreateTableCommand(), new CreateUserCommand(),
@@ -357,43 +355,44 @@ public class Shell {
new GetSplitsCommand(), new GrantCommand(), new GrepCommand(), new HelpCommand(), new HiddenCommand(), new HistoryCommand(),
new ImportDirectoryCommand(), new InfoCommand(), new InsertCommand(), new ListIterCommand(), new ListScansCommand(), new MasterStateCommand(),
new MaxRowCommand(), new MergeCommand(), new NoTableCommand(), new OfflineCommand(), new OnlineCommand(), new PasswdCommand(), new QuestionCommand(),
- new QuitCommand(), new RenameTableCommand(), new RevokeCommand(), new ScanCommand(), new SelectCommand(), new SelectrowCommand(),
- new SetAuthsCommand(), new SetGroupsCommand(), new SetIterCommand(), new SetScanIterCommand(), new SleepCommand(), new SystemPermissionsCommand(),
- new TableCommand(), new TablePermissionsCommand(), new TablesCommand(), new TraceCommand(), new UserCommand(), new UserPermissionsCommand(),
- new UsersCommand(), new WhoAmICommand(),};
+ new QuitCommand(), new RenameTableCommand(), new RevokeCommand(), new ScanCommand(), new org.apache.accumulo.core.util.shell.commands.SelectCommand(),
+ new org.apache.accumulo.core.util.shell.commands.SelectrowCommand(), new SetAuthsCommand(), new SetGroupsCommand(), new SetIterCommand(),
+ new SetScanIterCommand(), new SleepCommand(), new SystemPermissionsCommand(), new TableCommand(), new TablePermissionsCommand(), new TablesCommand(),
+ new TraceCommand(), new UserCommand(), new UserPermissionsCommand(), new UsersCommand(), new WhoAmICommand(),};
for (Command cmd : external) {
commandFactory.put(cmd.getName(), cmd);
}
}
-
+
/**
* @deprecated Not for client use
*/
+ @Deprecated
private static Instance getDefaultInstance(AccumuloConfiguration conf) {
String keepers = conf.get(Property.INSTANCE_ZK_HOST);
Path instanceDir = new Path(conf.get(Property.INSTANCE_DFS_DIR), "instance_id");
return new ZooKeeperInstance(UUID.fromString(ZooKeeperInstance.getInstanceIDFromHdfs(instanceDir)), keepers);
}
-
+
public Connector getConnector() {
return connector;
}
-
+
public static void main(String args[]) throws IOException {
Shell shell = new Shell();
shell.config(args);
-
+
System.exit(shell.start());
}
-
+
public int start() throws IOException {
if (configError)
return 1;
-
+
String input;
if (isVerbose())
printInfo();
-
+
String configDir = System.getenv("HOME") + "/.accumulo";
String historyPath = configDir + "/shell_history.txt";
File accumuloDir = new File(configDir);
@@ -406,9 +405,9 @@ public class Shell {
} catch (IOException e) {
log.warn("Unable to load history file at " + historyPath);
}
-
+
ShellCompletor userCompletor = null;
-
+
if (execFile != null) {
java.util.Scanner scanner = new java.util.Scanner(new File(execFile));
while (scanner.hasNextLine())
@@ -419,37 +418,37 @@ public class Shell {
}
return exitCode;
}
-
+
while (true) {
if (exit)
return exitCode;
-
+
// If tab completion is true we need to reset
if (tabCompletion) {
if (userCompletor != null)
reader.removeCompletor(userCompletor);
-
+
userCompletor = setupCompletion();
reader.addCompletor(userCompletor);
}
-
+
reader.setDefaultPrompt(getDefaultPrompt());
input = reader.readLine();
if (input == null) {
reader.printNewline();
return exitCode;
} // user canceled
-
+
execCommand(input, disableAuthTimeout, false);
}
}
-
+
public void printInfo() throws IOException {
reader.printString("\n" + SHELL_DESCRIPTION + "\n" + "- \n" + "- version: " + Constants.VERSION + "\n" + "- instance name: "
+ connector.getInstance().getInstanceName() + "\n" + "- instance id: " + connector.getInstance().getInstanceID() + "\n" + "- \n"
+ "- type 'help' for a list of available commands\n" + "- \n");
}
-
+
public void printVerboseInfo() throws IOException {
StringBuilder sb = new StringBuilder("-\n");
sb.append("- Current user: ").append(connector.whoami()).append("\n");
@@ -476,11 +475,11 @@ public class Shell {
sb.append("-\n");
reader.printString(sb.toString());
}
-
+
private String getDefaultPrompt() {
return connector.whoami() + "@" + connector.getInstance().getInstanceName() + (getTableName().isEmpty() ? "" : " ") + getTableName() + "> ";
}
-
+
public void execCommand(String input, boolean ignoreAuthTimeout, boolean echoPrompt) throws IOException {
audit.log(AuditLevel.AUDIT, getDefaultPrompt() + input);
if (echoPrompt) {
@@ -488,7 +487,7 @@ public class Shell {
reader.printString(input);
reader.printNewline();
}
-
+
String fields[];
try {
fields = new QuotedStringTokenizer(input).getTokens();
@@ -499,10 +498,10 @@ public class Shell {
}
if (fields.length == 0)
return;
-
+
String command = fields[0];
fields = fields.length > 1 ? Arrays.copyOfRange(fields, 1, fields.length) : new String[] {};
-
+
Command sc = null;
if (command.length() > 0) {
try {
@@ -512,7 +511,7 @@ public class Shell {
reader.printString(String.format("Unknown command \"%s\". Enter \"help\" for a list possible commands.\n", command));
return;
}
-
+
if (!(sc instanceof ExitCommand) && !ignoreAuthTimeout && System.currentTimeMillis() - lastUserActivity > authTimeout) {
reader.printString("Shell has been idle for too long. Please re-authenticate.\n");
boolean authFailed = true;
@@ -522,26 +521,26 @@ public class Shell {
reader.printNewline();
return;
} // user canceled
-
+
try {
authFailed = !connector.securityOperations().authenticateUser(connector.whoami(), pwd.getBytes());
} catch (Exception e) {
++exitCode;
printException(e);
}
-
+
if (authFailed)
reader.printString("Invalid password. ");
} while (authFailed);
lastUserActivity = System.currentTimeMillis();
}
-
+
// Get the options from the command on how to parse the string
Options parseOpts = sc.getOptionsWithHelp();
-
+
// Parse the string using the given options
CommandLine cl = new BasicParser().parse(parseOpts, fields);
-
+
int actualArgLen = cl.getArgs().length;
int expectedArgLen = sc.numArgs();
if (cl.hasOption(helpOption)) {
@@ -560,7 +559,7 @@ public class Shell {
exitCode += tmpCode;
reader.flushConsole();
}
-
+
} catch (ConstraintViolationException e) {
++exitCode;
printConstraintViolationException(e);
@@ -587,14 +586,14 @@ public class Shell {
printException(new BadArgumentException("Unrecognized empty command", command, -1));
}
}
-
+
/**
* The command tree is built in reverse so that the references are more easily linked up. There is some code in token to allow forward building of the command
* tree.
*/
private ShellCompletor setupCompletion() {
rootToken = new Token();
-
+
Set<String> tableNames = null;
try {
tableNames = connector.tableOperations().list();
@@ -602,7 +601,7 @@ public class Shell {
log.debug("Unable to obtain list of tables", e);
tableNames = Collections.emptySet();
}
-
+
Set<String> userlist = null;
try {
userlist = connector.securityOperations().listUsers();
@@ -610,25 +609,25 @@ public class Shell {
log.debug("Unable to obtain list of users", e);
userlist = Collections.emptySet();
}
-
+
Map<Command.CompletionSet,Set<String>> options = new HashMap<Command.CompletionSet,Set<String>>();
-
+
Set<String> commands = new HashSet<String>();
for (String a : commandFactory.keySet())
commands.add(a);
-
+
Set<String> modifiedUserlist = new HashSet<String>();
Set<String> modifiedTablenames = new HashSet<String>();
-
+
for (String a : tableNames)
modifiedTablenames.add(a.replaceAll("([\\s'\"])", "\\\\$1"));
for (String a : userlist)
modifiedUserlist.add(a.replaceAll("([\\s'\"])", "\\\\$1"));
-
+
options.put(Command.CompletionSet.USERNAMES, modifiedUserlist);
options.put(Command.CompletionSet.TABLENAMES, modifiedTablenames);
options.put(Command.CompletionSet.COMMANDS, commands);
-
+
for (Command c : commandFactory.values()) {
c.getOptionsWithHelp(); // prep the options for the command
// so that the completion can
@@ -637,7 +636,7 @@ public class Shell {
}
return new ShellCompletor(rootToken, options);
}
-
+
/**
* The Command class represents a command to be run in the shell. It contains the methods to execute along with some methods to help tab completion, and
* return the command name, help, and usage.
@@ -647,50 +646,50 @@ public class Shell {
public enum CompletionSet {
TABLENAMES, USERNAMES, COMMANDS
}
-
+
static Set<String> getCommandNames(Map<CompletionSet,Set<String>> objects) {
return objects.get(CompletionSet.COMMANDS);
}
-
+
static Set<String> getTableNames(Map<CompletionSet,Set<String>> objects) {
return objects.get(CompletionSet.TABLENAMES);
}
-
+
static Set<String> getUserNames(Map<CompletionSet,Set<String>> objects) {
return objects.get(CompletionSet.USERNAMES);
}
-
+
public void registerCompletionGeneral(Token root, Set<String> args, boolean caseSens) {
Token t = new Token(args);
t.setCaseSensitive(caseSens);
-
+
Token command = new Token(getName());
command.addSubcommand(t);
-
+
root.addSubcommand(command);
}
-
+
public void registerCompletionForTables(Token root, Map<CompletionSet,Set<String>> completionSet) {
registerCompletionGeneral(root, completionSet.get(CompletionSet.TABLENAMES), true);
}
-
+
public void registerCompletionForUsers(Token root, Map<CompletionSet,Set<String>> completionSet) {
registerCompletionGeneral(root, completionSet.get(CompletionSet.USERNAMES), true);
}
-
+
public void registerCompletionForCommands(Token root, Map<CompletionSet,Set<String>> completionSet) {
registerCompletionGeneral(root, completionSet.get(CompletionSet.COMMANDS), false);
}
-
+
// abstract methods to override
public abstract int execute(String fullCommand, CommandLine cl, Shell shellState) throws Exception;
-
+
public abstract String description();
-
+
public abstract int numArgs();
-
+
// OPTIONAL methods to override:
-
+
// the general version of getname uses reflection to get the class name
// and then cuts off the suffix -Command to get the name of the command
public String getName() {
@@ -699,58 +698,59 @@ public class Shell {
int i = s.indexOf("Command");
return i > 0 ? s.substring(st + 1, i).toLowerCase(Locale.ENGLISH) : null;
}
-
+
// The general version of this method adds the name
// of the command to the completion tree
public void registerCompletion(Token root, Map<CompletionSet,Set<String>> completion_set) {
root.addSubcommand(new Token(getName()));
}
-
+
// The general version of this method uses the HelpFormatter
// that comes with the apache Options package to print out the help
public final void printHelp() {
Shell.printHelp(usage(), "description: " + this.description(), getOptionsWithHelp());
}
-
+
public final void printHelp(int width) {
Shell.printHelp(usage(), "description: " + this.description(), getOptionsWithHelp(), width);
}
-
+
// Get options with help
public final Options getOptionsWithHelp() {
Options opts = getOptions();
opts.addOption(new Option(helpOption, helpLongOption, false, "display this help"));
return opts;
}
-
+
// General usage is just the command
public String usage() {
return getName();
}
-
+
// General Options are empty
public Options getOptions() {
return new Options();
}
}
-
+
public static abstract class TableOperation extends Command {
-
+
private Option optTablePattern, optTableName;
-
+
+ @Override
public int execute(String fullCommand, CommandLine cl, Shell shellState) throws Exception {
-
+
String originalTable = "";
-
+
if (shellState.getTableName() != "") {
originalTable = shellState.getTableName();
}
-
+
if (cl.hasOption(optTableName.getOpt())) {
String tableName = cl.getOptionValue(optTableName.getOpt());
shellState.setTableName(tableName);
}
-
+
try {
// populate the tablesToFlush set with the tables you want to flush
SortedSet<String> tablesToFlush = new TreeSet<String>();
@@ -761,15 +761,15 @@ public class Shell {
} else if (cl.hasOption(optTableName.getOpt())) {
tablesToFlush.add(cl.getOptionValue(optTableName.getOpt()));
}
-
+
else {
shellState.checkTableState();
tablesToFlush.add(shellState.getTableName());
}
-
+
if (tablesToFlush.isEmpty())
log.warn("No tables found that match your criteria");
-
+
// flush the tables
for (String tableName : tablesToFlush) {
if (!shellState.getConnector().tableOperations().exists(tableName))
@@ -777,60 +777,61 @@ public class Shell {
doTableOp(shellState, tableName);
}
}
-
+
finally {
shellState.setTableName(originalTable);
}
-
+
return 0;
}
-
+
protected abstract void doTableOp(Shell shellState, String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
-
+
@Override
public String description() {
return "makes a best effort to flush tables from memory to disk";
}
-
+
@Override
public Options getOptions() {
Options o = new Options();
-
+
optTablePattern = new Option("p", "pattern", true, "regex pattern of table names to flush");
optTablePattern.setArgName("pattern");
-
+
optTableName = new Option(tableOption, "table", true, "name of a table to flush");
optTableName.setArgName("tableName");
-
+
OptionGroup opg = new OptionGroup();
-
+
opg.addOption(optTablePattern);
opg.addOption(optTableName);
-
+
o.addOptionGroup(opg);
-
+
return o;
}
-
+
@Override
public int numArgs() {
return 0;
}
}
-
+
public interface PrintLine {
public void print(String s);
-
+
public void close();
}
-
+
public static class PrintShell implements PrintLine {
ConsoleReader reader;
-
+
public PrintShell(ConsoleReader reader) {
this.reader = reader;
}
-
+
+ @Override
public void print(String s) {
try {
reader.printString(s + "\n");
@@ -838,33 +839,36 @@ public class Shell {
throw new RuntimeException(ex);
}
}
-
+
+ @Override
public void close() {}
};
-
+
public static class PrintFile implements PrintLine {
PrintWriter writer;
-
+
public PrintFile(String filename) throws FileNotFoundException {
writer = new PrintWriter(filename);
}
-
+
+ @Override
public void print(String s) {
writer.println(s);
}
-
+
+ @Override
public void close() {
writer.close();
}
};
-
+
public final void printLines(Iterator<String> lines, boolean paginate) throws IOException {
int linesPrinted = 0;
String prompt = "-- hit any key to continue or 'q' to quit --";
int lastPromptLength = prompt.length();
int termWidth = reader.getTermwidth();
int maxLines = reader.getTermheight();
-
+
String peek = null;
while (lines.hasNext()) {
String nextLine = lines.next();
@@ -876,7 +880,7 @@ public class Shell {
reader.printNewline();
if (paginate) {
linesPrinted += peek.length() == 0 ? 0 : Math.ceil(peek.length() * 1.0 / termWidth);
-
+
// check if displaying the next line would result in
// scrolling off the screen
if (linesPrinted + Math.ceil(lastPromptLength * 1.0 / termWidth) + Math.ceil(prompt.length() * 1.0 / termWidth)
@@ -905,35 +909,35 @@ public class Shell {
reader.printNewline();
}
}
-
+
public final void printRecords(Iterable<Entry<Key,Value>> scanner, boolean printTimestamps, boolean paginate) throws IOException {
Class<? extends Formatter> formatterClass = getFormatter();
-
+
printRecords(scanner, printTimestamps, paginate, formatterClass);
}
-
+
public final void printRecords(Iterable<Entry<Key,Value>> scanner, boolean printTimestamps, boolean paginate, Class<? extends Formatter> formatterClass)
throws IOException {
printLines(FormatterFactory.getFormatter(formatterClass, scanner, printTimestamps), paginate);
}
-
+
public final void printBinaryRecords(Iterable<Entry<Key,Value>> scanner, boolean printTimestamps, boolean paginate) throws IOException {
printLines(FormatterFactory.getFormatter(binaryFormatterClass, scanner, printTimestamps), paginate);
}
-
+
public static String repeat(String s, int c) {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < c; i++)
sb.append(s);
return sb.toString();
}
-
+
public void checkTableState() {
if (getTableName().isEmpty())
throw new IllegalStateException(
"Not in a table context. Please use 'table <tableName>' to switch to a table, or use '-t' to specify a table if option is available.");
}
-
+
private final void printConstraintViolationException(ConstraintViolationException cve) {
printException(cve, "");
int COL1 = 50, COL2 = 14;
@@ -945,63 +949,63 @@ public class Shell {
log.error(String.format("%-" + COL1 + "s | %" + COL2 + "d | %-" + col3 + "s\n", cvs.constrainClass, cvs.violationCode, cvs.violationDescription));
log.error(String.format("%" + COL1 + "s-+-%" + COL2 + "s-+-%" + col3 + "s\n", repeat("-", COL1), repeat("-", COL2), repeat("-", col3)));
}
-
+
public static final void printException(Exception e) {
printException(e, e.getMessage());
}
-
+
private static final void printException(Exception e, String msg) {
log.error(e.getClass().getName() + (msg != null ? ": " + msg : ""));
log.debug(e.getClass().getName() + (msg != null ? ": " + msg : ""), e);
}
-
+
public static final void setDebugging(boolean debuggingEnabled) {
Logger.getLogger(Constants.CORE_PACKAGE_NAME).setLevel(debuggingEnabled ? Level.TRACE : Level.INFO);
}
-
+
public static final boolean isDebuggingEnabled() {
return Logger.getLogger(Constants.CORE_PACKAGE_NAME).isTraceEnabled();
}
-
+
private static final void printHelp(String usage, String description, Options opts) {
printHelp(usage, description, opts, Integer.MAX_VALUE);
}
-
+
private static final void printHelp(String usage, String description, Options opts, int width) {
PrintWriter pw = new PrintWriter(System.err);
new HelpFormatter().printHelp(pw, width, usage, description, opts, 2, 5, null, true);
pw.flush();
}
-
+
public void setExit(boolean exit) {
this.exit = exit;
}
-
+
public boolean isVerbose() {
return verbose;
}
-
+
public void setTableName(String tableName) {
this.tableName = tableName;
}
-
+
public String getTableName() {
return tableName;
}
-
+
public ConsoleReader getReader() {
return reader;
}
-
+
public void updateUser(AuthInfo authInfo) throws AccumuloException, AccumuloSecurityException {
connector = instance.getConnector(authInfo);
credentials = authInfo;
}
-
+
public AuthInfo getCredentials() {
return credentials;
}
-
+
/**
* Return the formatter for the current table.
*
@@ -1010,7 +1014,7 @@ public class Shell {
public Class<? extends Formatter> getFormatter() {
return getFormatter(this.tableName);
}
-
+
/**
* Return the formatter for the given table.
*
@@ -1020,7 +1024,7 @@ public class Shell {
*/
public Class<? extends Formatter> getFormatter(String tableName) {
Class<? extends Formatter> formatter = FormatterCommand.getCurrentFormatter(tableName, this);
-
+
if (null == formatter) {
log.error("Could not load the specified formatter. Using the DefaultFormatter");
return this.defaultFormatterClass;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SelectCommand.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SelectCommand.java b/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SelectCommand.java
index 5b18bca..7cf434b 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SelectCommand.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SelectCommand.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.io.Text;
/**
* @deprecated since 1.4, use {@code scan -r <row> -c <columnfamily>[:<columnqualifier>]} instead.
*/
+@Deprecated
public class SelectCommand extends Command {
private Option selectOptAuths, timestampOpt, disablePaginationOpt, tableOpt;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SelectrowCommand.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SelectrowCommand.java b/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SelectrowCommand.java
index 981596c..4037b3d 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SelectrowCommand.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SelectrowCommand.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.io.Text;
/**
* @deprecated since 1.4, use {@code scan -r <row>} instead.
*/
+@Deprecated
public class SelectrowCommand extends Command {
private Option selectrowOptAuths, timestampOpt, disablePaginationOpt, tableOpt;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/test/java/org/apache/accumulo/core/client/admin/TableOperationsHelperTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/client/admin/TableOperationsHelperTest.java b/src/core/src/test/java/org/apache/accumulo/core/client/admin/TableOperationsHelperTest.java
index 805904a..b9a69e8 100644
--- a/src/core/src/test/java/org/apache/accumulo/core/client/admin/TableOperationsHelperTest.java
+++ b/src/core/src/test/java/org/apache/accumulo/core/client/admin/TableOperationsHelperTest.java
@@ -191,6 +191,7 @@ public class TableOperationsHelperTest {
/**
* @deprecated since 1.4 {@link #attachIterator(String, IteratorSetting)}
*/
+ @Deprecated
@Override
public void addAggregators(String tableName, List<? extends PerColumnIteratorConfig> aggregators) throws AccumuloSecurityException, TableNotFoundException,
AccumuloException {}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java b/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
index f2e2f2c..c58bac8 100644
--- a/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
+++ b/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
@@ -31,15 +31,14 @@ import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.mapreduce.InputFormatBase.AccumuloIterator;
import org.apache.accumulo.core.client.mapreduce.InputFormatBase.AccumuloIteratorOption;
-import org.apache.accumulo.core.client.mapreduce.InputFormatBase.RegexType;
import org.apache.accumulo.core.client.mock.MockInstance;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.user.WholeRowIterator;
import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.core.util.ContextFactory;
+import org.apache.accumulo.core.util.Pair;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
@@ -100,7 +99,7 @@ public class AccumuloInputFormatTest {
@Test
public void testSetIterator() {
JobContext job = ContextFactory.createJobContext();
-
+
AccumuloInputFormat.setIterator(job, 1, "org.apache.accumulo.core.iterators.WholeRowIterator", "WholeRow");
Configuration conf = job.getConfiguration();
String iterators = conf.get("AccumuloInputFormat.iterators");
@@ -110,7 +109,7 @@ public class AccumuloInputFormatTest {
@Test
public void testAddIterator() {
JobContext job = ContextFactory.createJobContext();
-
+
AccumuloInputFormat.addIterator(job.getConfiguration(), new IteratorSetting(1, "WholeRow", WholeRowIterator.class));
AccumuloInputFormat.addIterator(job.getConfiguration(), new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
IteratorSetting iter = new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator");
@@ -192,7 +191,7 @@ public class AccumuloInputFormatTest {
@Test
public void testGetIteratorSettings() {
JobContext job = ContextFactory.createJobContext();
-
+
AccumuloInputFormat.setIterator(job, 1, "org.apache.accumulo.core.iterators.WholeRowIterator", "WholeRow");
AccumuloInputFormat.setIterator(job, 2, "org.apache.accumulo.core.iterators.VersioningIterator", "Versions");
AccumuloInputFormat.setIterator(job, 3, "org.apache.accumulo.core.iterators.CountingIterator", "Count");
@@ -241,7 +240,7 @@ public class AccumuloInputFormatTest {
@Test
public void testGetIteratorOption() {
JobContext job = ContextFactory.createJobContext();
-
+
AccumuloInputFormat.setIteratorOption(job, "iterator1", "key1", "value1");
AccumuloInputFormat.setIteratorOption(job, "iterator2", "key2", "value2");
AccumuloInputFormat.setIteratorOption(job, "iterator3", "key3", "value3");
@@ -272,12 +271,12 @@ public class AccumuloInputFormatTest {
@Test
public void testSetRegex() {
JobContext job = ContextFactory.createJobContext();
-
+
String regex = ">\"*%<>\'\\";
- AccumuloInputFormat.setRegex(job, RegexType.ROW, regex);
+ AccumuloInputFormat.setRegex(job, org.apache.accumulo.core.client.mapreduce.InputFormatBase.RegexType.ROW, regex);
- assertTrue(regex.equals(AccumuloInputFormat.getRegex(job, RegexType.ROW)));
+ assertTrue(regex.equals(AccumuloInputFormat.getRegex(job, org.apache.accumulo.core.client.mapreduce.InputFormatBase.RegexType.ROW)));
}
static class TestMapper extends Mapper<Key,Value,Key,Value> {
@@ -327,7 +326,7 @@ public class AccumuloInputFormatTest {
Assert.assertEquals("testtable", risplit.getTable());
Assert.assertEquals(new Authorizations(), risplit.getAuths());
Assert.assertEquals("testmapinstance", risplit.getInstanceName());
-
+
TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
RecordReader<Key,Value> reader = input.createRecordReader(split, tac);
Mapper<Key,Value,Key,Value>.Context context = ContextFactory.createMapContext(mapper, tac, reader, null, split);
@@ -348,7 +347,7 @@ public class AccumuloInputFormatTest {
bw.addMutation(m);
}
bw.close();
-
+
JobContext job = ContextFactory.createJobContext();
AccumuloInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "testtable2", new Authorizations());
AccumuloInputFormat.setMockInstance(job.getConfiguration(), "testmapinstance");
@@ -362,7 +361,7 @@ public class AccumuloInputFormatTest {
Mapper<Key,Value,Key,Value>.Context context = ContextFactory.createMapContext(mapper, tac, rr, null, ris);
rr.initialize(ris, tac);
while (rr.nextKeyValue()) {
- mapper.map(rr.getCurrentKey(), rr.getCurrentValue(), (TestMapper.Context) context);
+ mapper.map(rr.getCurrentKey(), rr.getCurrentValue(), context);
}
}
@@ -379,12 +378,12 @@ public class AccumuloInputFormatTest {
bw.addMutation(m);
}
bw.close();
-
+
JobContext job = ContextFactory.createJobContext();
AccumuloInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "testtable3", new Authorizations());
AccumuloInputFormat.setMockInstance(job.getConfiguration(), "testmapinstance");
final String regex = ".*1.*";
- AccumuloInputFormat.setRegex(job, RegexType.ROW, regex);
+ AccumuloInputFormat.setRegex(job, org.apache.accumulo.core.client.mapreduce.InputFormatBase.RegexType.ROW, regex);
AccumuloInputFormat input = new AccumuloInputFormat();
RangeInputSplit ris = new RangeInputSplit();
TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
@@ -417,28 +416,28 @@ public class AccumuloInputFormatTest {
AccumuloInputFormat.setInputInfo(job, username, password, table, auths);
AccumuloInputFormat.setMockInstance(job, instance);
- AccumuloInputFormat.setRegex(job, RegexType.ROW, rowRegex);
- AccumuloInputFormat.setRegex(job, RegexType.COLUMN_FAMILY, colfRegex);
- AccumuloInputFormat.setRegex(job, RegexType.COLUMN_QUALIFIER, colqRegex);
- AccumuloInputFormat.setRegex(job, RegexType.VALUE, valRegex);
+ AccumuloInputFormat.setRegex(job, org.apache.accumulo.core.client.mapreduce.InputFormatBase.RegexType.ROW, rowRegex);
+ AccumuloInputFormat.setRegex(job, org.apache.accumulo.core.client.mapreduce.InputFormatBase.RegexType.COLUMN_FAMILY, colfRegex);
+ AccumuloInputFormat.setRegex(job, org.apache.accumulo.core.client.mapreduce.InputFormatBase.RegexType.COLUMN_QUALIFIER, colqRegex);
+ AccumuloInputFormat.setRegex(job, org.apache.accumulo.core.client.mapreduce.InputFormatBase.RegexType.VALUE, valRegex);
AccumuloInputFormat.setIsolated(job, isolated);
AccumuloInputFormat.setLocalIterators(job, localIters);
AccumuloInputFormat.setMaxVersions(job, maxVersions);
AccumuloInputFormat.fetchColumns(job, fetchColumns);
AccumuloInputFormat.setLogLevel(job, level);
-
+
AccumuloInputFormat aif = new AccumuloInputFormat();
-
+
List<InputSplit> splits = aif.getSplits(job);
-
+
Assert.assertEquals(1, splits.size());
-
+
InputSplit split = splits.get(0);
-
+
Assert.assertEquals(RangeInputSplit.class, split.getClass());
-
+
RangeInputSplit risplit = (RangeInputSplit) split;
-
+
Assert.assertEquals(username, risplit.getUsername());
Assert.assertEquals(table, risplit.getTable());
Assert.assertArrayEquals(password, risplit.getPassword());
@@ -480,9 +479,9 @@ public class AccumuloInputFormatTest {
assertEquals(splits.size(), 1);
TestMapper mapper = (TestMapper) job.getMapperClass().newInstance();
-
+
RangeInputSplit emptySplit = new RangeInputSplit();
-
+
// Using an empty split should fall back to the information in the Job's Configuration
TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
RecordReader<Key,Value> reader = input.createRecordReader(emptySplit, tac);
@@ -516,11 +515,11 @@ public class AccumuloInputFormatTest {
assertEquals(splits.size(), 1);
TestMapper mapper = (TestMapper) job.getMapperClass().newInstance();
-
+
RangeInputSplit emptySplit = new RangeInputSplit();
emptySplit.setUsername("root");
emptySplit.setPassword("anythingelse".getBytes());
-
+
// Using an empty split should fall back to the information in the Job's Configuration
TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
RecordReader<Key,Value> reader = input.createRecordReader(emptySplit, tac);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java b/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java
index 94e8bf0..2e6ba76 100644
--- a/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java
+++ b/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java
@@ -50,7 +50,7 @@ public class AccumuloOutputFormatTest {
static class TestMapper extends Mapper<Key,Value,Text,Mutation> {
Key key = null;
int count = 0;
-
+
@Override
protected void map(Key k, Value v, Context context) throws IOException, InterruptedException {
if (key != null)
@@ -60,7 +60,7 @@ public class AccumuloOutputFormatTest {
key = new Key(k);
count++;
}
-
+
@Override
protected void cleanup(Context context) throws IOException, InterruptedException {
super.cleanup(context);
@@ -71,7 +71,7 @@ public class AccumuloOutputFormatTest {
} catch (NullPointerException e) {}
}
}
-
+
@Test
public void testMR() throws Exception {
MockInstance mockInstance = new MockInstance("testmrinstance");
@@ -85,7 +85,7 @@ public class AccumuloOutputFormatTest {
bw.addMutation(m);
}
bw.close();
-
+
Job job = new Job();
job.setInputFormatClass(AccumuloInputFormat.class);
job.setMapperClass(TestMapper.class);
@@ -95,15 +95,15 @@ public class AccumuloOutputFormatTest {
job.setNumReduceTasks(0);
AccumuloInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "testtable1", new Authorizations());
AccumuloInputFormat.setMockInstance(job.getConfiguration(), "testmrinstance");
- AccumuloOutputFormat.setOutputInfo(job, "root", "".getBytes(), false, "testtable2");
- AccumuloOutputFormat.setMockInstance(job, "testmrinstance");
-
+ AccumuloOutputFormat.setOutputInfo(job.getConfiguration(), "root", "".getBytes(), false, "testtable2");
+ AccumuloOutputFormat.setMockInstance(job.getConfiguration(), "testmrinstance");
+
AccumuloInputFormat input = new AccumuloInputFormat();
List<InputSplit> splits = input.getSplits(job);
assertEquals(splits.size(), 1);
-
+
AccumuloOutputFormat output = new AccumuloOutputFormat();
-
+
TestMapper mapper = (TestMapper) job.getMapperClass().newInstance();
for (InputSplit split : splits) {
TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
@@ -114,7 +114,7 @@ public class AccumuloOutputFormatTest {
mapper.run(context);
writer.close(context);
}
-
+
Scanner scanner = c.createScanner("testtable2", new Authorizations());
Iterator<Entry<Key,Value>> iter = scanner.iterator();
assertTrue(iter.hasNext());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/test/java/org/apache/accumulo/core/data/MapFileTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/data/MapFileTest.java b/src/core/src/test/java/org/apache/accumulo/core/data/MapFileTest.java
index 258fc38..9e60f27 100644
--- a/src/core/src/test/java/org/apache/accumulo/core/data/MapFileTest.java
+++ b/src/core/src/test/java/org/apache/accumulo/core/data/MapFileTest.java
@@ -41,6 +41,7 @@ import org.apache.log4j.Logger;
/**
* @deprecated since 1.4
*/
+@Deprecated
public class MapFileTest extends TestCase {
private static final Logger log = Logger.getLogger(MapFileTest.class);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/test/java/org/apache/accumulo/core/iterators/AggregatingIteratorTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/iterators/AggregatingIteratorTest.java b/src/core/src/test/java/org/apache/accumulo/core/iterators/AggregatingIteratorTest.java
index 80a099c..137a462 100644
--- a/src/core/src/test/java/org/apache/accumulo/core/iterators/AggregatingIteratorTest.java
+++ b/src/core/src/test/java/org/apache/accumulo/core/iterators/AggregatingIteratorTest.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.io.Text;
/**
* @deprecated since 1.4
*/
+@Deprecated
public class AggregatingIteratorTest extends TestCase {
private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/test/java/org/apache/accumulo/core/iterators/FilteringIteratorTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/iterators/FilteringIteratorTest.java b/src/core/src/test/java/org/apache/accumulo/core/iterators/FilteringIteratorTest.java
index e2a2994..370244c 100644
--- a/src/core/src/test/java/org/apache/accumulo/core/iterators/FilteringIteratorTest.java
+++ b/src/core/src/test/java/org/apache/accumulo/core/iterators/FilteringIteratorTest.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.io.Text;
/**
* @deprecated since 1.4
*/
+@Deprecated
public class FilteringIteratorTest extends TestCase {
private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/test/java/org/apache/accumulo/core/iterators/RegExIteratorTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/iterators/RegExIteratorTest.java b/src/core/src/test/java/org/apache/accumulo/core/iterators/RegExIteratorTest.java
index bb8e3bb..5c7969e 100644
--- a/src/core/src/test/java/org/apache/accumulo/core/iterators/RegExIteratorTest.java
+++ b/src/core/src/test/java/org/apache/accumulo/core/iterators/RegExIteratorTest.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.io.Text;
/**
* @deprecated since 1.4
*/
+@Deprecated
public class RegExIteratorTest extends TestCase {
private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/test/java/org/apache/accumulo/core/iterators/aggregation/NumSummationTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/iterators/aggregation/NumSummationTest.java b/src/core/src/test/java/org/apache/accumulo/core/iterators/aggregation/NumSummationTest.java
index 7e1a333..61e18b8 100644
--- a/src/core/src/test/java/org/apache/accumulo/core/iterators/aggregation/NumSummationTest.java
+++ b/src/core/src/test/java/org/apache/accumulo/core/iterators/aggregation/NumSummationTest.java
@@ -23,6 +23,7 @@ import org.apache.accumulo.core.data.Value;
/**
* @deprecated since 1.4
*/
+@Deprecated
public class NumSummationTest extends TestCase {
public byte[] init(int n) {
byte[] b = new byte[n];
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/test/java/org/apache/accumulo/core/iterators/aggregation/conf/AggregatorConfigurationTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/iterators/aggregation/conf/AggregatorConfigurationTest.java b/src/core/src/test/java/org/apache/accumulo/core/iterators/aggregation/conf/AggregatorConfigurationTest.java
index f92cc73..b17ff28 100644
--- a/src/core/src/test/java/org/apache/accumulo/core/iterators/aggregation/conf/AggregatorConfigurationTest.java
+++ b/src/core/src/test/java/org/apache/accumulo/core/iterators/aggregation/conf/AggregatorConfigurationTest.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.io.Text;
/**
* @deprecated since 1.4
*/
+@Deprecated
public class AggregatorConfigurationTest extends TestCase {
public void testBinary() {
Text colf = new Text();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/test/java/org/apache/accumulo/core/iterators/filter/ColumnFilterTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/iterators/filter/ColumnFilterTest.java b/src/core/src/test/java/org/apache/accumulo/core/iterators/filter/ColumnFilterTest.java
index 4568667..2f3d7fa 100644
--- a/src/core/src/test/java/org/apache/accumulo/core/iterators/filter/ColumnFilterTest.java
+++ b/src/core/src/test/java/org/apache/accumulo/core/iterators/filter/ColumnFilterTest.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.io.Text;
/**
* @deprecated since 1.4
*/
+@Deprecated
public class ColumnFilterTest extends TestCase {
Key nk(String row, String cf, String cq) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java b/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
index 18054ca..3c18be9 100644
--- a/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
+++ b/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
@@ -796,6 +796,7 @@ public class CombinerTest {
* @throws IOException
* @deprecated since 1.4
*/
+ @Deprecated
public void testCombinerCompatibility() throws IOException {
long[] la = {1l, 2l, 3l};
List<Long> ll = new ArrayList<Long>(Arrays.asList((Long) 1l, (Long) 2l, (Long) 3l));
@@ -811,6 +812,7 @@ public class CombinerTest {
* @throws IOException
* @deprecated since 1.4
*/
+ @Deprecated
public void testLongEncoding(long l) throws IOException {
assertEquals((Long) l, SummingCombiner.FIXED_LEN_ENCODER.decode(LongSummation.longToBytes(l)));
assertEquals(l, LongSummation.bytesToLong(SummingCombiner.FIXED_LEN_ENCODER.encode(l)));
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
----------------------------------------------------------------------
diff --git a/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java b/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
index 3480be9..1efd0d4 100644
--- a/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
+++ b/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
@@ -27,13 +27,13 @@ import java.util.Map;
*/
public class MiniAccumuloConfig {
-
+
private File dir = null;
private String rootPassword = null;
private Map<String,String> siteConfig = Collections.emptyMap();
private int numTservers = 2;
private boolean runGC = false;
-
+
/**
* @param dir
* An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
@@ -41,42 +41,42 @@ public class MiniAccumuloConfig {
* @param rootPassword
* The initial password for the Accumulo root user
*/
-
+
public MiniAccumuloConfig(File dir, String rootPassword) {
this.dir = dir;
this.rootPassword = rootPassword;
}
-
+
public File getDir() {
return dir;
}
-
+
public String getRootPassword() {
return rootPassword;
}
-
+
public int getNumTservers() {
return numTservers;
}
-
+
/**
* Calling this method is optional. If not set, it defaults to two.
*
* @param numTservers
* the number of tablet servers that mini accumulo cluster should start
*/
-
+
public MiniAccumuloConfig setNumTservers(int numTservers) {
if (numTservers < 1)
throw new IllegalArgumentException("Must have at least one tablet server");
this.numTservers = numTservers;
return this;
}
-
+
public Map<String,String> getSiteConfig() {
return siteConfig;
}
-
+
/**
* Calling this method is optional. If not set, it defautls to an empty map.
*
@@ -87,17 +87,17 @@ public class MiniAccumuloConfig {
this.siteConfig = siteConfig;
return this;
}
-
+
/**
* Whether or not the Accumulo garbage collector proces will run
- * @return
*/
public boolean shouldRunGC() {
return runGC;
}
-
+
/**
* Sets if the Accumulo garbage collector process should run
+ *
* @param shouldRunGC
*/
public void runGC(boolean shouldRunGC) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/proxy/pom.xml
----------------------------------------------------------------------
diff --git a/src/proxy/pom.xml b/src/proxy/pom.xml
index e1daa29..eb88e6b 100644
--- a/src/proxy/pom.xml
+++ b/src/proxy/pom.xml
@@ -99,7 +99,6 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>13.0.1</version>
<scope>compile</scope>
</dependency>
<dependency>
@@ -116,7 +115,6 @@
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-server</artifactId>
- <version>${project.version}</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/server/src/main/java/org/apache/accumulo/server/conf/TableConfiguration.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/conf/TableConfiguration.java b/src/server/src/main/java/org/apache/accumulo/server/conf/TableConfiguration.java
index 935abe3..5ca3179 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/conf/TableConfiguration.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/conf/TableConfiguration.java
@@ -56,6 +56,7 @@ public class TableConfiguration extends AccumuloConfiguration {
/**
* @deprecated not for client use
*/
+ @Deprecated
private static ZooCache getTablePropCache() {
Instance inst = HdfsZooInstance.getInstance();
if (tablePropCache == null)
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java b/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
index d83c898..bf8c160 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
@@ -45,14 +45,13 @@ public class LogReader {
public static void usage() {
System.err.println("Usage : " + LogReader.class.getName() + " [-r <row>] [-m <maxColumns] [-t <key extent>] [-p <row regex>] <log file>");
}
-
+
/**
* Dump a Log File (Map or Sequence) to stdout. Will read from HDFS or local file system.
*
* @param args
* - first argument is the file to print
* @throws IOException
- * @throws ParseException
*/
public static void main(String[] args) throws IOException {
Configuration conf = CachedConfiguration.getInstance();
@@ -74,7 +73,7 @@ public class LogReader {
usage();
return;
}
-
+
Matcher rowMatcher = null;
KeyExtent ke = null;
Text row = null;
@@ -97,15 +96,15 @@ public class LogReader {
Pattern pattern = Pattern.compile(cl.getOptionValue(rowPatternOpt.getOpt()));
rowMatcher = pattern.matcher("");
}
-
+
Set<Integer> tabletIds = new HashSet<Integer>();
for (String file : files) {
-
+
Path path = new Path(file);
LogFileKey key = new LogFileKey();
LogFileValue value = new LogFileValue();
-
+
if (fs.isFile(path)) {
// read log entries from a simple hdfs file
org.apache.hadoop.io.SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(file), conf);
@@ -134,9 +133,9 @@ public class LogReader {
}
}
}
-
+
public static void printLogEvent(LogFileKey key, LogFileValue value, Text row, Matcher rowMatcher, KeyExtent ke, Set<Integer> tabletIds, int maxMutations) {
-
+
if (ke != null) {
if (key.event == LogEvents.DEFINE_TABLET) {
if (key.tablet.equals(ke)) {
@@ -157,7 +156,7 @@ public class LogReader {
found = true;
break;
}
-
+
if (rowMatcher != null) {
rowMatcher.reset(new String(m.getRow()));
if (rowMatcher.matches()) {
@@ -166,17 +165,17 @@ public class LogReader {
}
}
}
-
+
if (!found)
return;
} else {
return;
}
-
+
}
-
+
System.out.println(key);
System.out.println(LogFileValue.format(value, maxMutations));
}
-
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/server/src/main/java/org/apache/accumulo/server/logger/LogWriter.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/logger/LogWriter.java b/src/server/src/main/java/org/apache/accumulo/server/logger/LogWriter.java
index 4ed9af9..a8bc87c 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/logger/LogWriter.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/logger/LogWriter.java
@@ -75,51 +75,50 @@ import org.apache.hadoop.io.WritableName;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.thrift.TException;
-
/**
* Write log operations to open {@link org.apache.hadoop.io.SequenceFile}s referenced by log id's.
*/
class LogWriter implements MutationLogger.Iface {
static final org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger(LogWriter.class);
-
+
static {
WritableName.setName(LogFileKey.class, Constants.OLD_PACKAGE_NAME + ".server.logger.LogFileKey");
WritableName.setName(LogFileValue.class, Constants.OLD_PACKAGE_NAME + ".server.logger.LogFileValue");
}
-
+
static class LogWriteException extends RuntimeException {
/**
*
*/
private static final long serialVersionUID = 1L;
-
+
public LogWriteException(Throwable why) {
super(why);
}
}
-
+
/**
* Generate log ids.
*/
private static final SecureRandom random = new SecureRandom();
-
+
private final FileSystem fs;
private final ExecutorService copyThreadPool;
private final static Mutation empty[] = new Mutation[0];
private boolean closed = false;
-
+
private static class Logger {
SequenceFile.Writer seq;
FSDataOutputStream out;
LogFileKey key = new LogFileKey();
LogFileValue value = new LogFileValue();
-
+
Logger(Configuration conf, String path) throws IOException {
FileSystem local = TraceFileSystem.wrap(FileSystem.getLocal(conf).getRaw());
out = local.create(new Path(path));
seq = SequenceFile.createWriter(conf, out, LogFileKey.class, LogFileValue.class, CompressionType.NONE, null);
}
-
+
void logEntry() throws IOException {
try {
long t1 = System.currentTimeMillis();
@@ -137,44 +136,44 @@ class LogWriter implements MutationLogger.Iface {
throw ioe;
}
}
-
+
void close() throws IOException {
seq.close();
out.close();
}
}
-
+
/**
* The current loggers in use.
*/
private Map<Long,Logger> logs = new ConcurrentHashMap<Long,Logger>();
-
+
/**
* Map from filename to open log id.
*/
private Map<String,Long> file2id = new ConcurrentHashMap<String,Long>();
-
+
/**
* Local directory where logs are created.
*/
private final List<String> roots;
private int nextRoot = 0;
-
+
private final String instanceId;
-
+
private LogArchiver logArchiver;
-
+
// Metrics MBean
private static LogWriterMetrics metrics = new LogWriterMetrics();
-
+
private final AccumuloConfiguration acuConf;
-
+
/**
*
* @param fs
* The HDFS instance shared by master/tservers.
- * @param logDirectory
- * The local directory to write the recovery logs.
+ * @param logDirectories
+ * The local directories to write the recovery logs.
* @param instanceId
* The accumulo instance for which we are logging.
*/
@@ -189,7 +188,7 @@ class LogWriter implements MutationLogger.Iface {
} catch (IOException e1) {
throw new RuntimeException(e1);
}
-
+
// Register the metrics MBean
try {
metrics.register();
@@ -197,7 +196,7 @@ class LogWriter implements MutationLogger.Iface {
log.error("Exception registering MBean with MBean Server", e);
}
}
-
+
@Override
synchronized public void close(TInfo info, long id) throws NoSuchLogIDException {
long t1 = System.currentTimeMillis();
@@ -224,7 +223,7 @@ class LogWriter implements MutationLogger.Iface {
}
throw new RuntimeException("Unexpected failure to find a filename matching an id");
}
-
+
private final String findLocalFilename(String localLog) throws FileNotFoundException {
for (String root : roots) {
String path = root + "/" + localLog;
@@ -233,7 +232,7 @@ class LogWriter implements MutationLogger.Iface {
}
throw new FileNotFoundException("No file " + localLog + " found in " + roots);
}
-
+
@Override
public LogCopyInfo startCopy(TInfo info, AuthInfo credentials, final String localLog, final String fullyQualifiedFileName, final boolean sort) {
log.info("Copying " + localLog + " to " + fullyQualifiedFileName);
@@ -254,7 +253,7 @@ class LogWriter implements MutationLogger.Iface {
throw new RuntimeException(ex);
}
long result = file.length();
-
+
copyThreadPool.execute(new Runnable() {
@Override
public void run() {
@@ -282,16 +281,16 @@ class LogWriter implements MutationLogger.Iface {
if (metrics.isEnabled())
metrics.add(LogWriterMetrics.copy, (t2 - t1));
}
-
+
private void copySortLog(String localLog, String fullyQualifiedFileName) throws IOException {
final long SORT_BUFFER_SIZE = acuConf.getMemoryInBytes(Property.LOGGER_SORT_BUFFER_SIZE);
-
+
FileSystem local = TraceFileSystem.wrap(FileSystem.getLocal(fs.getConf()).getRaw());
Path dest = new Path(fullyQualifiedFileName + ".recovered");
log.debug("Sorting log file to DSF " + dest);
fs.mkdirs(dest);
int part = 0;
-
+
Reader reader = new SequenceFile.Reader(local, new Path(findLocalFilename(localLog)), fs.getConf());
try {
final ArrayList<Pair<LogFileKey,LogFileValue>> kv = new ArrayList<Pair<LogFileKey,LogFileValue>>();
@@ -323,7 +322,7 @@ class LogWriter implements MutationLogger.Iface {
reader.close();
}
}
-
+
private void writeSortedEntries(Path dest, int part, final List<Pair<LogFileKey,LogFileValue>> kv) throws IOException {
String path = dest + String.format("/part-r-%05d", part);
log.debug("Writing partial log file to DSF " + path);
@@ -351,7 +350,7 @@ class LogWriter implements MutationLogger.Iface {
span.stop();
}
}
-
+
private void copyLog(final String localLog, final String fullyQualifiedFileName) throws IOException {
Path dest = new Path(fullyQualifiedFileName + ".copy");
log.debug("Copying log file to DSF " + dest);
@@ -384,7 +383,7 @@ class LogWriter implements MutationLogger.Iface {
});
return new LogCopyInfo(result, null);
}
-
+
@Override
public LogFile create(TInfo info, AuthInfo credentials, String tserverSession) throws ThriftSecurityException {
if (closed)
@@ -422,13 +421,13 @@ class LogWriter implements MutationLogger.Iface {
log.info("Created log " + result.name);
return result;
}
-
+
@Override
public void log(TInfo info, long id, final long seq, final int tid, final TMutation mutation) throws NoSuchLogIDException {
Logger out = logs.get(id);
if (out == null)
throw new NoSuchLogIDException();
-
+
out.key.event = MUTATION;
out.key.seq = seq;
out.key.tid = tid;
@@ -446,7 +445,7 @@ class LogWriter implements MutationLogger.Iface {
throw new LogWriteException(e);
}
}
-
+
private void logMany(TInfo info, long id, final long seq, final int tid, Mutation muations[]) throws NoSuchLogIDException {
Logger out = logs.get(id);
if (out == null)
@@ -467,7 +466,7 @@ class LogWriter implements MutationLogger.Iface {
throw new LogWriteException(e);
}
}
-
+
@Override
public void minorCompactionFinished(TInfo info, long id, final long seq, final int tid, final String fqfn) throws NoSuchLogIDException {
Logger out = logs.get(id);
@@ -490,7 +489,7 @@ class LogWriter implements MutationLogger.Iface {
throw new LogWriteException(e);
}
}
-
+
@Override
public void minorCompactionStarted(TInfo info, long id, final long seq, final int tid, final String fqfn) throws NoSuchLogIDException {
Logger out = logs.get(id);
@@ -513,7 +512,7 @@ class LogWriter implements MutationLogger.Iface {
throw new LogWriteException(e);
}
}
-
+
@Override
public void defineTablet(TInfo info, long id, final long seq, final int tid, final TKeyExtent tablet) throws NoSuchLogIDException {
Logger out = logs.get(id);
@@ -536,7 +535,7 @@ class LogWriter implements MutationLogger.Iface {
throw new LogWriteException(e);
}
}
-
+
public void shutdown() {
// Make sure everything is closed
log.info("Shutting down");
@@ -550,7 +549,7 @@ class LogWriter implements MutationLogger.Iface {
}
}
}
-
+
@Override
public List<String> getClosedLogs(TInfo info, AuthInfo credentials) throws ThriftSecurityException {
ArrayList<String> result = new ArrayList<String>();
@@ -574,7 +573,7 @@ class LogWriter implements MutationLogger.Iface {
}
return result;
}
-
+
@Override
public void remove(TInfo info, AuthInfo credentials, List<String> files) {
log.info("Deleting " + files.size() + " log files");
@@ -590,10 +589,10 @@ class LogWriter implements MutationLogger.Iface {
log.error("Unable to delete files", ex);
}
}
-
+
@Override
public void logManyTablets(TInfo info, long id, List<TabletMutations> mutations) throws NoSuchLogIDException {
-
+
for (TabletMutations tm : mutations) {
Mutation ma[] = new Mutation[tm.mutations.size()];
int index = 0;
@@ -602,7 +601,7 @@ class LogWriter implements MutationLogger.Iface {
logMany(info, id, tm.seq, tm.tabletID, ma);
}
}
-
+
@Override
synchronized public void beginShutdown(TInfo tinfo, AuthInfo credentials) throws TException {
closed = true;
@@ -618,8 +617,8 @@ class LogWriter implements MutationLogger.Iface {
file2id.clear();
}
}
-
+
@Override
public void halt(TInfo tinfo, AuthInfo credentials) throws TException {}
-
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/server/src/main/java/org/apache/accumulo/server/master/LogSort.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/master/LogSort.java b/src/server/src/main/java/org/apache/accumulo/server/master/LogSort.java
index 4666331..1c384a3 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/master/LogSort.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/master/LogSort.java
@@ -54,31 +54,31 @@ import org.apache.log4j.Logger;
/* Borrows from the Apache sort example program */
public class LogSort extends Configured implements Tool {
-
+
private static final Logger log = Logger.getLogger(LogSort.class);
public static final String INSTANCE_ID_PROPERTY = "accumulo.instance.id";
private Job job = null;
-
+
public static String getJobName() {
return "LogSort_" + HdfsZooInstance.getInstance().getInstanceID();
}
-
+
private void printUsage() {
System.out.println("accumulo " + this.getClass().getName() + " [-r <reducers>] [-q queue] [-p pool] <input> <output>");
ToolRunner.printGenericCommandUsage(System.out);
}
-
+
public static class SortCommit extends FileOutputCommitter {
-
+
final private Path outputPath;
final private FileSystem outputFileSystem;
-
+
public SortCommit(Path outputPath, TaskAttemptContext context) throws IOException {
super(outputPath, context);
this.outputPath = outputPath;
outputFileSystem = outputPath.getFileSystem(context.getConfiguration());
}
-
+
@Override
public void abortTask(TaskAttemptContext context) {
try {
@@ -88,7 +88,8 @@ public class LogSort extends Configured implements Tool {
throw new RuntimeException(ex);
}
}
-
+
+ @Deprecated
@Override
public void cleanupJob(JobContext context) throws IOException {
super.cleanupJob(context);
@@ -106,14 +107,15 @@ public class LogSort extends Configured implements Tool {
}
}
}
-
+
/**
* The main driver for sort program. Invoke this method to submit the map/reduce job.
*/
+ @Override
public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
if (job != null)
throw new RuntimeException("job has already run");
-
+
// parse options
int num_reduces = 1;
String queueName = "default";
@@ -139,38 +141,38 @@ public class LogSort extends Configured implements Tool {
return 1;
}
}
-
+
// validate arguments
if (otherArgs.size() != 2) {
log.error("Wrong number of parameters: " + otherArgs.size() + " instead of 2.", new Exception());
printUsage();
return 0;
}
-
+
// create job
job = new Job(getConf(), getJobName());
job.setJarByClass(this.getClass());
-
+
// set input
job.setInputFormatClass(SequenceFileInputFormat.class);
SequenceFileInputFormat.setInputPaths(job, otherArgs.get(0));
-
+
// set identity mappers
job.setMapperClass(Mapper.class);
job.setOutputKeyClass(LogFileKey.class);
job.setOutputValueClass(LogFileValue.class);
-
+
// set custom partitioner
job.setPartitionerClass(RoundRobinPartitioner.class);
-
+
// set identity reducer
job.setReducerClass(IdentityReducer.class);
job.setNumReduceTasks(num_reduces);
-
+
// set output
job.setOutputFormatClass(LoggerMapFileOutputFormat.class);
LoggerMapFileOutputFormat.setOutputPath(job, new Path(otherArgs.get(1)));
-
+
// submit the job to the job queue
job.getConfiguration().set("mapred.job.queue.name", queueName);
job.getConfiguration().set("mapred.job.pool.name", poolName);
@@ -179,7 +181,7 @@ public class LogSort extends Configured implements Tool {
+ " with " + num_reduces + " reduces.");
return 0;
}
-
+
public static Job startSort(boolean background, String[] args) throws Exception {
LogSort sort = new LogSort();
ToolRunner.run(CachedConfiguration.getInstance(), sort, args);
@@ -189,7 +191,7 @@ public class LogSort extends Configured implements Tool {
sort.job.waitForCompletion(true);
return sort.job;
}
-
+
public static void main(String[] args) throws Exception {
long startTime = System.currentTimeMillis();
log.info("Job started");
@@ -198,25 +200,25 @@ public class LogSort extends Configured implements Tool {
if (!job.isSuccessful())
System.exit(1);
}
-
+
private static class LoggerMapFileOutputFormat extends FileOutputFormat<WritableComparable<?>,Writable> {
@Override
public RecordWriter<WritableComparable<?>,Writable> getRecordWriter(final TaskAttemptContext job) throws IOException, InterruptedException {
// get the path of the temporary output file
Path file = getDefaultWorkFile(job, "");
-
+
FileSystem fs = file.getFileSystem(job.getConfiguration());
CompressionCodec codec = null;
CompressionType compressionType = CompressionType.NONE;
if (getCompressOutput(job)) {
// find the kind of compression to do
compressionType = SequenceFileOutputFormat.getOutputCompressionType(job);
-
+
// find the right codec
Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job, DefaultCodec.class);
codec = ReflectionUtils.newInstance(codecClass, job.getConfiguration());
}
-
+
Progressable progress = new Progressable() {
@Override
public void progress() {
@@ -226,20 +228,20 @@ public class LogSort extends Configured implements Tool {
final MapFile.Writer out = new MapFile.Writer(job.getConfiguration(), fs, file.toString(), job.getOutputKeyClass().asSubclass(WritableComparable.class),
job.getOutputValueClass().asSubclass(Writable.class), compressionType, codec, progress);
return new RecordWriter<WritableComparable<?>,Writable>() {
-
+
@Override
public void write(WritableComparable<?> key, Writable value) throws IOException {
out.append(key, value);
}
-
+
@Override
public void close(TaskAttemptContext context) throws IOException, InterruptedException {
out.close();
}
};
-
+
}
-
+
@Override
public synchronized OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException {
return new SortCommit(getOutputPath(context), context);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/trace/NullScanner.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/trace/NullScanner.java b/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/trace/NullScanner.java
index d80efcf..4de3de0 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/trace/NullScanner.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/trace/NullScanner.java
@@ -28,88 +28,95 @@ import org.apache.accumulo.core.data.Value;
import org.apache.hadoop.io.Text;
public class NullScanner implements Scanner {
-
+
+ @Deprecated
@Override
public void setScanIterators(int priority, String iteratorClass, String iteratorName) {}
-
+
@Override
public void addScanIterator(IteratorSetting cfg) {}
-
+
+ @Deprecated
@Override
public void setScanIteratorOption(String iteratorName, String key, String value) {}
-
+
@Override
public void updateScanIteratorOption(String iteratorName, String key, String value) {}
-
+
+ @Deprecated
@Override
public void setupRegex(String iteratorName, int iteratorPriority) throws IOException {}
-
+
+ @Deprecated
@Override
public void setRowRegex(String regex) {}
-
+
+ @Deprecated
@Override
public void setColumnFamilyRegex(String regex) {}
-
+
+ @Deprecated
@Override
public void setColumnQualifierRegex(String regex) {}
-
+
+ @Deprecated
@Override
public void setValueRegex(String regex) {}
-
+
@Override
public void fetchColumnFamily(Text col) {}
-
+
@Override
public void fetchColumn(Text colFam, Text colQual) {}
-
+
@Override
public void clearColumns() {}
-
+
@Override
public void clearScanIterators() {}
-
+
@Override
public void setTimeOut(int timeOut) {}
-
+
@Override
public int getTimeOut() {
return 0;
}
-
+
@Override
public void setRange(Range range) {}
-
+
@Override
public Range getRange() {
return null;
}
-
+
@Override
public void setBatchSize(int size) {
-
+
}
-
+
@Override
public int getBatchSize() {
return 0;
}
-
+
@Override
public void enableIsolation() {
-
+
}
-
+
@Override
public void disableIsolation() {
-
+
}
-
+
@Override
public Iterator<Entry<Key,Value>> iterator() {
return new NullKeyValueIterator();
}
-
+
@Override
public void removeScanIterator(String iteratorName) {}
-
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/server/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java b/src/server/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
index 995b6bb..94dcd1b 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
@@ -18,7 +18,6 @@ package org.apache.accumulo.server.security;
import java.io.IOException;
import java.net.InetAddress;
-import java.net.UnknownHostException;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
@@ -34,9 +33,6 @@ public class SecurityUtil {
/**
* This method is for logging a server in kerberos. If this is used in client code, it will fail unless run as the accumulo keytab's owner. Instead, use
* {@link #login(String, String)}
- *
- * @throws UnknownHostException
- * @throws IOException
*/
public static void serverLogin() {
@SuppressWarnings("deprecation")
@@ -69,7 +65,6 @@ public class SecurityUtil {
* replaced by the systems host name.
* @param keyTabPath
* @return true if login succeeded, otherwise false
- * @throws IOException
*/
public static boolean login(String principalConfig, String keyTabPath) {
try {
[5/5] git commit: ACCUMULO-1961 Fix warnings
Posted by ct...@apache.org.
ACCUMULO-1961 Fix warnings
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a3a88955
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a3a88955
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a3a88955
Branch: refs/heads/1.4.5-SNAPSHOT
Commit: a3a889554940d249504a2b03c0e6dda2a25ef5cd
Parents: 47403ba
Author: Christopher Tubbs <ct...@apache.org>
Authored: Tue Dec 3 15:12:27 2013 -0500
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Tue Dec 3 15:26:56 2013 -0500
----------------------------------------------------------------------
pom.xml | 55 +-
.../apache/accumulo/core/client/Connector.java | 2 +
.../accumulo/core/client/IteratorSetting.java | 57 +-
.../accumulo/core/client/ScannerBase.java | 7 +
.../core/client/admin/TableOperations.java | 3 +
.../core/client/admin/TableOperationsImpl.java | 3 +
.../core/client/impl/ConnectorImpl.java | 1 +
.../core/client/impl/OfflineScanner.java | 131 ++-
.../core/client/impl/ScannerOptions.java | 7 +
.../client/impl/TabletServerBatchDeleter.java | 11 +-
.../mapreduce/AccumuloFileOutputFormat.java | 19 +-
.../client/mapreduce/AccumuloOutputFormat.java | 18 +
.../core/client/mapreduce/InputFormatBase.java | 34 +
.../accumulo/core/client/mock/MockAccumulo.java | 1 +
.../accumulo/core/client/mock/MockTable.java | 1 +
.../core/client/mock/MockTableOperations.java | 1 +
.../core/conf/AccumuloConfiguration.java | 1 +
.../org/apache/accumulo/core/data/Value.java | 6 +
.../accumulo/core/file/BloomFilterLayer.java | 201 ++--
.../core/file/map/MapFileOperations.java | 1 +
.../accumulo/core/file/map/MapFileUtil.java | 2 +
.../accumulo/core/file/map/MyMapFile.java | 2 +
.../accumulo/core/file/map/MySequenceFile.java | 1 +
.../accumulo/core/file/rfile/RelativeKey.java | 222 ++--
.../accumulo/core/file/rfile/bcfile/BCFile.java | 3 -
.../accumulo/core/file/rfile/bcfile/TFile.java | 404 +++----
.../core/iterators/AggregatingIterator.java | 1 +
.../iterators/FamilyIntersectingIterator.java | 1 +
.../core/iterators/FilteringIterator.java | 1 +
.../core/iterators/FirstEntryInRowIterator.java | 1 +
.../accumulo/core/iterators/GrepIterator.java | 1 +
.../core/iterators/IntersectingIterator.java | 1 +
.../accumulo/core/iterators/IteratorUtil.java | 1 +
.../accumulo/core/iterators/LargeRowFilter.java | 1 +
.../core/iterators/NoLabelIterator.java | 1 +
.../accumulo/core/iterators/RegExIterator.java | 1 +
.../core/iterators/RowDeletingIterator.java | 1 +
.../core/iterators/SortedKeyValueIterator.java | 6 +-
.../core/iterators/VersioningIterator.java | 1 +
.../core/iterators/WholeRowIterator.java | 1 +
.../core/iterators/aggregation/Aggregator.java | 1 +
.../iterators/aggregation/LongSummation.java | 1 +
.../aggregation/NumArraySummation.java | 1 +
.../iterators/aggregation/NumSummation.java | 1 +
.../core/iterators/aggregation/StringMax.java | 1 +
.../core/iterators/aggregation/StringMin.java | 1 +
.../iterators/aggregation/StringSummation.java | 1 +
.../conf/AggregatorConfiguration.java | 1 +
.../aggregation/conf/AggregatorSet.java | 1 +
.../iterators/conf/PerColumnIteratorConfig.java | 1 +
.../core/iterators/filter/AgeOffFilter.java | 1 +
.../iterators/filter/ColumnAgeOffFilter.java | 1 +
.../iterators/filter/ColumnQualifierFilter.java | 1 +
.../core/iterators/filter/DeleteFilter.java | 1 +
.../accumulo/core/iterators/filter/Filter.java | 1 +
.../core/iterators/filter/NoLabelFilter.java | 1 +
.../core/iterators/filter/RegExFilter.java | 1 +
.../core/iterators/filter/VisibilityFilter.java | 1 +
.../iterators/user/IntersectingIterator.java | 3 +
.../apache/accumulo/core/util/shell/Shell.java | 332 +++---
.../core/util/shell/commands/SelectCommand.java | 1 +
.../util/shell/commands/SelectrowCommand.java | 1 +
.../client/admin/TableOperationsHelperTest.java | 1 +
.../mapreduce/AccumuloInputFormatTest.java | 57 +-
.../mapreduce/AccumuloOutputFormatTest.java | 20 +-
.../apache/accumulo/core/data/MapFileTest.java | 1 +
.../core/iterators/AggregatingIteratorTest.java | 1 +
.../core/iterators/FilteringIteratorTest.java | 1 +
.../core/iterators/RegExIteratorTest.java | 1 +
.../iterators/aggregation/NumSummationTest.java | 1 +
.../conf/AggregatorConfigurationTest.java | 1 +
.../core/iterators/filter/ColumnFilterTest.java | 1 +
.../core/iterators/user/CombinerTest.java | 2 +
.../minicluster/MiniAccumuloConfig.java | 26 +-
src/proxy/pom.xml | 2 -
.../server/conf/TableConfiguration.java | 1 +
.../accumulo/server/logger/LogReader.java | 25 +-
.../accumulo/server/logger/LogWriter.java | 89 +-
.../apache/accumulo/server/master/LogSort.java | 58 +-
.../monitor/servlets/trace/NullScanner.java | 61 +-
.../accumulo/server/security/SecurityUtil.java | 5 -
.../server/tabletserver/TabletServer.java | 1047 +++++++++---------
.../server/test/CreateRandomMapFile.java | 1 +
.../accumulo/server/test/DumpMapFile.java | 1 +
.../apache/accumulo/server/test/MMFITest.java | 1 +
.../accumulo/server/test/MidPointPerfTest2.java | 2 +
.../accumulo/server/util/CountDiskRows.java | 1 +
.../accumulo/server/util/DumpMapFile.java | 1 +
.../apache/accumulo/server/util/DumpTable.java | 1 +
.../server/util/MapFilePerformanceTest.java | 1 +
.../server/util/OfflineMetadataScanner.java | 95 +-
.../accumulo/server/util/InitializeTest.java | 11 +-
92 files changed, 1633 insertions(+), 1451 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d0b3041..9d33818 100644
--- a/pom.xml
+++ b/pom.xml
@@ -14,7 +14,8 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
---><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
@@ -22,8 +23,8 @@
<artifactId>apache</artifactId>
<version>10</version>
</parent>
-
-
+
+
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo</artifactId>
<packaging>pom</packaging>
@@ -134,7 +135,7 @@
<include>web/**</include>
<include>randomwalk/**</include>
<include>*.*</include>
- <include>**/META-INF/*</include>
+ <include>**/META-INF/*</include>
</includes>
</configuration>
</plugin>
@@ -196,7 +197,7 @@
<projversion>${project.version}</projversion>
<summary>Apache Accumulo BigTable clone</summary>
<description>
- Apache Accumulo is a large distributed structured store based on
+ Apache Accumulo is a large distributed structured store based on
Google's BigTable design.
</description>
<copyright>2011 The Apache Software Foundation.</copyright>
@@ -280,7 +281,7 @@
</mapping>
<mapping>
<directory>/opt/accumulo/accumulo-${project.version}/lib</directory>
- <dependency/>
+ <dependency />
<sources>
<source>
<location>lib</location>
@@ -343,7 +344,7 @@
<type>file</type>
<mapper>
<type>perm</type>
- <prefix>/usr/lib/accumulo</prefix>
+ <prefix>/usr/lib/accumulo</prefix>
</mapper>
</data>
<data>
@@ -351,7 +352,7 @@
<type>file</type>
<mapper>
<type>perm</type>
- <prefix>/usr/lib/accumulo</prefix>
+ <prefix>/usr/lib/accumulo</prefix>
</mapper>
</data>
<data>
@@ -359,7 +360,7 @@
<type>file</type>
<mapper>
<type>perm</type>
- <prefix>/usr/lib/accumulo</prefix>
+ <prefix>/usr/lib/accumulo</prefix>
</mapper>
</data>
<data>
@@ -367,7 +368,7 @@
<type>file</type>
<mapper>
<type>perm</type>
- <prefix>/usr/lib/accumulo</prefix>
+ <prefix>/usr/lib/accumulo</prefix>
</mapper>
</data>
<data>
@@ -506,7 +507,20 @@
</goals>
</pluginExecutionFilter>
<action>
- <ignore/>
+ <ignore />
+ </action>
+ </pluginExecution>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-remote-resources-plugin</artifactId>
+ <versionRange>[1.0,)</versionRange>
+ <goals>
+ <goal>process</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore />
</action>
</pluginExecution>
<pluginExecution>
@@ -519,7 +533,20 @@
</goals>
</pluginExecutionFilter>
<action>
- <ignore/>
+ <ignore />
+ </action>
+ </pluginExecution>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <versionRange>[1.0,)</versionRange>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore />
</action>
</pluginExecution>
</pluginExecutions>
@@ -761,11 +788,11 @@
<artifactId>accumulo-server</artifactId>
<version>1.4.5-SNAPSHOT</version>
</dependency>
- <dependency>
+ <dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-minicluster</artifactId>
<version>1.4.5-SNAPSHOT</version>
- </dependency>
+ </dependency>
<!-- additional dependencies we need to provide -->
<dependency>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/client/Connector.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/Connector.java b/src/core/src/main/java/org/apache/accumulo/core/client/Connector.java
index 72b3252..5f8a3f4 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/Connector.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/Connector.java
@@ -46,6 +46,7 @@ public class Connector {
* when a user's credentials are invalid
* @see Instance#getConnector(String user, byte[] password)
*/
+ @Deprecated
public Connector(Instance instance, String user, byte[] password) throws AccumuloException, AccumuloSecurityException {
impl = instance.getConnector(user, password);
}
@@ -54,6 +55,7 @@ public class Connector {
* @see Instance#getConnector(String user, byte[] password)
* @deprecated Not for client use
*/
+ @Deprecated
public Connector() {
impl = null;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java b/src/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
index e815072..d30ea37 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
@@ -47,7 +47,7 @@ public class IteratorSetting {
private String name;
private String iteratorClass;
private Map<String,String> properties;
-
+
/**
* Get layer at which this iterator applies. See {@link #setPriority(int)} for how the priority is used.
*
@@ -56,7 +56,7 @@ public class IteratorSetting {
public int getPriority() {
return priority;
}
-
+
/**
* Set layer at which this iterator applies.
*
@@ -68,7 +68,7 @@ public class IteratorSetting {
ArgumentChecker.strictlyPositive(priority);
this.priority = priority;
}
-
+
/**
* Get the iterator's name.
*
@@ -77,7 +77,7 @@ public class IteratorSetting {
public String getName() {
return name;
}
-
+
/**
* Set the iterator's name. Must be a simple alphanumeric identifier.
*
@@ -87,7 +87,7 @@ public class IteratorSetting {
ArgumentChecker.notNull(name);
this.name = name;
}
-
+
/**
* Get the name of the class that implements the iterator.
*
@@ -96,7 +96,7 @@ public class IteratorSetting {
public String getIteratorClass() {
return iteratorClass;
}
-
+
/**
* Set the name of the class that implements the iterator. The class does not have to be present on the client, but it must be available to all tablet
* servers.
@@ -107,7 +107,7 @@ public class IteratorSetting {
ArgumentChecker.notNull(iteratorClass);
this.iteratorClass = iteratorClass;
}
-
+
/**
* Constructs an iterator setting configured for the scan scope with no parameters. (Parameters can be added later.)
*
@@ -121,7 +121,7 @@ public class IteratorSetting {
public IteratorSetting(int priority, String name, String iteratorClass) {
this(priority, name, iteratorClass, new HashMap<String,String>());
}
-
+
/**
* Constructs an iterator setting configured for the specified scopes with the specified parameters.
*
@@ -141,7 +141,7 @@ public class IteratorSetting {
this.properties = new HashMap<String,String>();
addOptions(properties);
}
-
+
/**
* Constructs an iterator setting using the given class's SimpleName for the iterator name. The iterator setting will be configured for the scan scope with no
* parameters.
@@ -154,7 +154,7 @@ public class IteratorSetting {
public IteratorSetting(int priority, Class<? extends SortedKeyValueIterator<Key,Value>> iteratorClass) {
this(priority, iteratorClass.getSimpleName(), iteratorClass.getName());
}
-
+
/**
*
* Constructs an iterator setting using the given class's SimpleName for the iterator name and configured for the specified scopes with the specified
@@ -170,7 +170,7 @@ public class IteratorSetting {
public IteratorSetting(int priority, Class<? extends SortedKeyValueIterator<Key,Value>> iteratorClass, Map<String,String> properties) {
this(priority, iteratorClass.getSimpleName(), iteratorClass.getName(), properties);
}
-
+
/**
* Constructs an iterator setting configured for the scan scope with no parameters.
*
@@ -184,7 +184,7 @@ public class IteratorSetting {
public IteratorSetting(int priority, String name, Class<? extends SortedKeyValueIterator<Key,Value>> iteratorClass) {
this(priority, name, iteratorClass.getName());
}
-
+
/**
* Add another option to the iterator.
*
@@ -197,7 +197,7 @@ public class IteratorSetting {
ArgumentChecker.notNull(option, value);
properties.put(option, value);
}
-
+
/**
* Remove an option from the iterator.
*
@@ -209,7 +209,7 @@ public class IteratorSetting {
ArgumentChecker.notNull(option);
return properties.remove(option);
}
-
+
/**
* Add many options to the iterator.
*
@@ -222,7 +222,7 @@ public class IteratorSetting {
addOption(keyValue.getKey(), keyValue.getValue());
}
}
-
+
/**
* Add many options to the iterator.
*
@@ -233,7 +233,7 @@ public class IteratorSetting {
ArgumentChecker.notNull(properties);
addOptions(properties.entrySet());
}
-
+
/**
* Get the configuration parameters for this iterator.
*
@@ -242,14 +242,14 @@ public class IteratorSetting {
public Map<String,String> getOptions() {
return Collections.unmodifiableMap(properties);
}
-
+
/**
* Remove all options from the iterator.
*/
public void clearOptions() {
properties.clear();
}
-
+
/**
* @see java.lang.Object#hashCode()
*/
@@ -263,10 +263,7 @@ public class IteratorSetting {
result = prime * result + ((properties == null) ? 0 : properties.hashCode());
return result;
}
-
- /**
- * @see java.lang.Object#equals()
- */
+
@Override
public boolean equals(Object obj) {
if (this == obj)
@@ -312,35 +309,35 @@ public class IteratorSetting {
sb.append(properties);
return sb.toString();
}
-
+
/**
* A convenience class for passing column family and column qualifiers to iterator configuration methods.
*/
public static class Column extends Pair<Text,Text> {
-
+
public Column(Text columnFamily, Text columnQualifier) {
super(columnFamily, columnQualifier);
}
-
+
public Column(Text columnFamily) {
super(columnFamily, null);
}
-
+
public Column(String columnFamily, String columnQualifier) {
super(new Text(columnFamily), new Text(columnQualifier));
}
-
+
public Column(String columnFamily) {
super(new Text(columnFamily), null);
}
-
+
public Text getColumnFamily() {
return getFirst();
}
-
+
public Text getColumnQualifier() {
return getSecond();
}
-
+
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java b/src/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
index 570ed6f..f0eb5c8 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
@@ -65,11 +65,13 @@ public interface ScannerBase extends Iterable<Entry<Key,Value>> {
/**
* @deprecated Since 1.4, use {@link org.apache.accumulo.core.client.ScannerBase#addScanIterator}.
*/
+ @Deprecated
public void setScanIterators(int priority, String iteratorClass, String iteratorName) throws IOException;
/**
* @deprecated Since 1.4, use {@link org.apache.accumulo.core.client.ScannerBase#addScanIterator}.
*/
+ @Deprecated
public void setScanIteratorOption(String iteratorName, String key, String value);
/**
@@ -85,6 +87,7 @@ public interface ScannerBase extends Iterable<Entry<Key,Value>> {
* @deprecated Since 1.4, use {@link org.apache.accumulo.core.client.ScannerBase#addScanIterator} and
* {@link org.apache.accumulo.core.iterators.user.RegExFilter}
*/
+ @Deprecated
public void setupRegex(String iteratorName, int iteratorPriority) throws IOException;
/**
@@ -104,6 +107,7 @@ public interface ScannerBase extends Iterable<Entry<Key,Value>> {
* scanner.addScanIterator(priority, cfg);
* </pre>
*/
+ @Deprecated
public void setRowRegex(String regex);
/**
@@ -116,6 +120,7 @@ public interface ScannerBase extends Iterable<Entry<Key,Value>> {
* @deprecated Since 1.4, use {@link org.apache.accumulo.core.client.ScannerBase#addScanIterator} and
* {@link org.apache.accumulo.core.iterators.user.RegExFilter}
*/
+ @Deprecated
public void setColumnFamilyRegex(String regex);
/**
@@ -130,6 +135,7 @@ public interface ScannerBase extends Iterable<Entry<Key,Value>> {
* {@link org.apache.accumulo.core.iterators.user.RegExFilter}
*
*/
+ @Deprecated
public void setColumnQualifierRegex(String regex);
/**
@@ -141,6 +147,7 @@ public interface ScannerBase extends Iterable<Entry<Key,Value>> {
* @deprecated Since 1.4, use {@link org.apache.accumulo.core.client.ScannerBase#addScanIterator} and
* {@link org.apache.accumulo.core.iterators.user.RegExFilter}
*/
+ @Deprecated
public void setValueRegex(String regex);
/**
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java b/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
index e696ce9..ac233e3 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
@@ -114,6 +114,7 @@ public interface TableOperations {
*
* @deprecated since 1.4 {@link #attachIterator(String, IteratorSetting)}
*/
+ @Deprecated
public void addAggregators(String tableName, List<? extends PerColumnIteratorConfig> aggregators) throws AccumuloSecurityException, TableNotFoundException,
AccumuloException;
@@ -282,6 +283,7 @@ public interface TableOperations {
*
* @deprecated As of release 1.4, replaced by {@link #flush(String, Text, Text, boolean)}
*/
+ @Deprecated
public void flush(String tableName) throws AccumuloException, AccumuloSecurityException;
/**
@@ -416,6 +418,7 @@ public interface TableOperations {
/**
* @deprecated Since 1.4 use {@link #importDirectory(String, String, String, boolean)}
*/
+ @Deprecated
public AssignmentStats importDirectory(String tableName, String dir, String failureDir, int numThreads, int numAssignThreads, boolean disableGC)
throws IOException, AccumuloException, AccumuloSecurityException;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java b/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
index 74b3498..adc95b6 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
@@ -560,6 +560,7 @@ public class TableOperationsImpl extends TableOperationsHelper {
/**
* @deprecated since 1.4 {@link #flush(String, Text, Text, boolean)}
*/
+ @Deprecated
public void flush(String tableName) throws AccumuloException, AccumuloSecurityException {
try {
flush(tableName, null, null, false);
@@ -898,6 +899,7 @@ public class TableOperationsImpl extends TableOperationsHelper {
/**
* @deprecated since 1.4 {@link #importDirectory(String, String, String, boolean)}
*/
+ @Deprecated
@Override
public AssignmentStats importDirectory(String tableName, String dir, String failureDir, int numThreads, int numAssignThreads, boolean disableGC)
throws IOException, AccumuloException, AccumuloSecurityException {
@@ -1043,6 +1045,7 @@ public class TableOperationsImpl extends TableOperationsHelper {
*
* @deprecated since 1.4 {@link #attachIterator(String, IteratorSetting)}
*/
+ @Deprecated
@Override
public void addAggregators(String tableName, List<? extends org.apache.accumulo.core.iterators.conf.PerColumnIteratorConfig> aggregators)
throws AccumuloSecurityException, TableNotFoundException,
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java b/src/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java
index adc2556..6404b5f 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java
@@ -60,6 +60,7 @@ public class ConnectorImpl extends Connector {
* @see Instance#getConnector(String user, byte[] password)
* @deprecated Not for client use
*/
+ @Deprecated
public ConnectorImpl(Instance instance, String user, byte[] password) throws AccumuloException, AccumuloSecurityException {
ArgumentChecker.notNull(instance, user, password);
this.instance = instance;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java b/src/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
index c26024c..d1f90b3 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
@@ -67,35 +67,35 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.Text;
class OfflineIterator implements Iterator<Entry<Key,Value>> {
-
+
static class OfflineIteratorEnvironment implements IteratorEnvironment {
@Override
public SortedKeyValueIterator<Key,Value> reserveMapFileReader(String mapFileName) throws IOException {
throw new NotImplementedException();
}
-
+
@Override
public AccumuloConfiguration getConfig() {
return AccumuloConfiguration.getDefaultConfiguration();
}
-
+
@Override
public IteratorScope getIteratorScope() {
return IteratorScope.scan;
}
-
+
@Override
public boolean isFullMajorCompaction() {
return false;
}
-
+
private ArrayList<SortedKeyValueIterator<Key,Value>> topLevelIterators = new ArrayList<SortedKeyValueIterator<Key,Value>>();
-
+
@Override
public void registerSideChannel(SortedKeyValueIterator<Key,Value> iter) {
topLevelIterators.add(iter);
}
-
+
SortedKeyValueIterator<Key,Value> getTopLevelIterator(SortedKeyValueIterator<Key,Value> iter) {
if (topLevelIterators.isEmpty())
return iter;
@@ -115,18 +115,11 @@ class OfflineIterator implements Iterator<Entry<Key,Value>> {
private ScannerOptions options;
private ArrayList<SortedKeyValueIterator<Key,Value>> readers;
- /**
- * @param offlineScanner
- * @param instance
- * @param credentials
- * @param authorizations
- * @param table
- */
public OfflineIterator(ScannerOptions options, Instance instance, AuthInfo credentials, Authorizations authorizations, Text table, Range range) {
this.options = new ScannerOptions(options);
this.instance = instance;
this.range = range;
-
+
if (this.options.fetchedColumns.size() > 0) {
this.range = range.bound(this.options.fetchedColumns.first(), this.options.fetchedColumns.last());
}
@@ -134,11 +127,11 @@ class OfflineIterator implements Iterator<Entry<Key,Value>> {
this.tableId = table.toString();
this.authorizations = authorizations;
this.readers = new ArrayList<SortedKeyValueIterator<Key,Value>>();
-
+
try {
conn = instance.getConnector(credentials);
nextTablet();
-
+
while (iter != null && !iter.hasTop())
nextTablet();
@@ -151,7 +144,7 @@ class OfflineIterator implements Iterator<Entry<Key,Value>> {
public boolean hasNext() {
return iter != null && iter.hasTop();
}
-
+
@Override
public Entry<Key,Value> next() {
try {
@@ -160,16 +153,16 @@ class OfflineIterator implements Iterator<Entry<Key,Value>> {
KeyValue ret = new KeyValue(new Key(iter.getTopKey()), Arrays.copyOf(v, v.length));
iter.next();
-
+
while (iter != null && !iter.hasTop())
nextTablet();
-
+
return ret;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
-
+
/**
* @throws TableNotFoundException
* @throws IOException
@@ -177,20 +170,20 @@ class OfflineIterator implements Iterator<Entry<Key,Value>> {
*
*/
private void nextTablet() throws TableNotFoundException, AccumuloException, IOException {
-
+
Range nextRange = null;
-
+
if (currentExtent == null) {
Text startRow;
-
+
if (range.getStartKey() != null)
startRow = range.getStartKey().getRow();
else
startRow = new Text();
-
+
nextRange = new Range(new KeyExtent(new Text(tableId), startRow, null).getMetadataEntry(), true, null, false);
} else {
-
+
if (currentExtent.getEndRow() == null) {
iter = null;
return;
@@ -205,7 +198,7 @@ class OfflineIterator implements Iterator<Entry<Key,Value>> {
}
List<String> relFiles = new ArrayList<String>();
-
+
Pair<KeyExtent,String> eloc = getTabletFiles(nextRange, relFiles);
while (eloc.getSecond() != null) {
@@ -215,21 +208,21 @@ class OfflineIterator implements Iterator<Entry<Key,Value>> {
throw new AccumuloException("Table is online " + tableId + " cannot scan tablet in offline mode " + eloc.getFirst());
}
}
-
+
UtilWaitThread.sleep(250);
-
+
eloc = getTabletFiles(nextRange, relFiles);
}
-
+
KeyExtent extent = eloc.getFirst();
-
+
if (!extent.getTableId().toString().equals(tableId)) {
throw new AccumuloException(" did not find tablets for table " + tableId + " " + extent);
}
if (currentExtent != null && !extent.isPreviousExtent(currentExtent))
throw new AccumuloException(" " + currentExtent + " is not previous extent " + extent);
-
+
String tablesDir = Constants.getTablesDir(instance.getConfiguration());
List<String> absFiles = new ArrayList<String>();
for (String relPath : relFiles) {
@@ -238,24 +231,24 @@ class OfflineIterator implements Iterator<Entry<Key,Value>> {
else
absFiles.add(tablesDir + "/" + tableId + relPath);
}
-
+
iter = createIterator(extent, absFiles);
iter.seek(range, LocalityGroupUtil.families(options.fetchedColumns), options.fetchedColumns.size() == 0 ? false : true);
currentExtent = extent;
-
+
}
-
+
private Pair<KeyExtent,String> getTabletFiles(Range nextRange, List<String> relFiles) throws TableNotFoundException {
Scanner scanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
scanner.setBatchSize(100);
scanner.setRange(nextRange);
-
+
RowIterator rowIter = new RowIterator(scanner);
Iterator<Entry<Key,Value>> row = rowIter.next();
-
+
KeyExtent extent = null;
String location = null;
-
+
while (row.hasNext()) {
Entry<Key,Value> entry = row.next();
Key key = entry.getKey();
@@ -263,12 +256,12 @@ class OfflineIterator implements Iterator<Entry<Key,Value>> {
if (key.getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
relFiles.add(key.getColumnQualifier().toString());
}
-
+
if (key.getColumnFamily().equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY)
|| key.getColumnFamily().equals(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY)) {
location = entry.getValue().toString();
}
-
+
if (Constants.METADATA_PREV_ROW_COLUMN.hasColumns(key)) {
extent = new KeyExtent(key.getRow(), entry.getValue());
}
@@ -286,18 +279,18 @@ class OfflineIterator implements Iterator<Entry<Key,Value>> {
*/
private SortedKeyValueIterator<Key,Value> createIterator(KeyExtent extent, List<String> absFiles) throws TableNotFoundException, AccumuloException,
IOException {
-
+
// TODO share code w/ tablet
AccumuloConfiguration acuTableConf = AccumuloConfiguration.getTableConfiguration(conn, tableId);
-
+
Configuration conf = CachedConfiguration.getInstance();
-
+
FileSystem fs = FileUtil.getFileSystem(conf, instance.getConfiguration());
-
+
for (SortedKeyValueIterator<Key,Value> reader : readers) {
((FileSKVIterator) reader).close();
}
-
+
readers.clear();
// TODO need to close files
@@ -305,24 +298,24 @@ class OfflineIterator implements Iterator<Entry<Key,Value>> {
FileSKVIterator reader = FileOperations.getInstance().openReader(file, false, fs, conf, acuTableConf, null, null);
readers.add(reader);
}
-
+
MultiIterator multiIter = new MultiIterator(readers, extent);
-
+
OfflineIteratorEnvironment iterEnv = new OfflineIteratorEnvironment();
-
+
DeletingIterator delIter = new DeletingIterator(multiIter, false);
-
+
ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(delIter);
-
+
ColumnQualifierFilter colFilter = new ColumnQualifierFilter(cfsi, new HashSet<Column>(options.fetchedColumns));
-
+
byte[] defaultSecurityLabel;
-
+
ColumnVisibility cv = new ColumnVisibility(acuTableConf.get(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY));
defaultSecurityLabel = cv.getExpression();
-
+
VisibilityFilter visFilter = new VisibilityFilter(colFilter, authorizations, defaultSecurityLabel);
-
+
return iterEnv.getTopLevelIterator(IteratorUtil.loadIterators(IteratorScope.scan, visFilter, extent, acuTableConf, options.serverSideIteratorList,
options.serverSideIteratorOptions, iterEnv, false));
}
@@ -331,23 +324,23 @@ class OfflineIterator implements Iterator<Entry<Key,Value>> {
public void remove() {
throw new UnsupportedOperationException();
}
-
+
}
/**
*
*/
public class OfflineScanner extends ScannerOptions implements Scanner {
-
+
private int batchSize;
private int timeOut;
private Range range;
-
+
private Instance instance;
private AuthInfo credentials;
private Authorizations authorizations;
private Text tableId;
-
+
public OfflineScanner(Instance instance, AuthInfo credentials, String tableId, Authorizations authorizations) {
ArgumentChecker.notNull(instance, credentials, tableId, authorizations);
this.instance = instance;
@@ -356,7 +349,7 @@ public class OfflineScanner extends ScannerOptions implements Scanner {
this.range = new Range((Key) null, (Key) null);
this.authorizations = authorizations;
-
+
this.batchSize = Constants.SCAN_BATCH_SIZE;
this.timeOut = Integer.MAX_VALUE;
}
@@ -365,42 +358,42 @@ public class OfflineScanner extends ScannerOptions implements Scanner {
public void setTimeOut(int timeOut) {
this.timeOut = timeOut;
}
-
+
@Override
public int getTimeOut() {
return timeOut;
}
-
+
@Override
public void setRange(Range range) {
this.range = range;
}
-
+
@Override
public Range getRange() {
return range;
}
-
+
@Override
public void setBatchSize(int size) {
this.batchSize = size;
}
-
+
@Override
public int getBatchSize() {
return batchSize;
}
-
+
@Override
public void enableIsolation() {
-
+
}
-
+
@Override
public void disableIsolation() {
-
+
}
-
+
@Override
public Iterator<Entry<Key,Value>> iterator() {
return new OfflineIterator(this, instance, credentials, authorizations, tableId, range);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/client/impl/ScannerOptions.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/impl/ScannerOptions.java b/src/core/src/main/java/org/apache/accumulo/core/client/impl/ScannerOptions.java
index 9233a3e..958b202 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/impl/ScannerOptions.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/impl/ScannerOptions.java
@@ -105,6 +105,7 @@ public class ScannerOptions implements ScannerBase {
/**
* @deprecated since 1.4, use {@link #addScanIterator(IteratorSetting)}
*/
+ @Deprecated
@Override
public void setScanIterators(int priority, String iteratorClass, String iteratorName) {
addScanIterator(new IteratorSetting(priority, iteratorName, iteratorClass));
@@ -113,6 +114,7 @@ public class ScannerOptions implements ScannerBase {
/**
* @deprecated since 1.4, use {@link #updateScanIteratorOption(String, String, String)}
*/
+ @Deprecated
@Override
public synchronized void setScanIteratorOption(String iteratorName, String key, String value) {
updateScanIteratorOption(iteratorName, key, value);
@@ -142,6 +144,7 @@ public class ScannerOptions implements ScannerBase {
* @deprecated since 1.4, use {@link #addScanIterator(IteratorSetting)}
* @see org.apache.accumulo.core.iterators.user.RegExFilter
*/
+ @Deprecated
@Override
public synchronized void setupRegex(String iteratorName, int iteratorPriority) throws IOException {
ArgumentChecker.notNull(iteratorName);
@@ -166,6 +169,7 @@ public class ScannerOptions implements ScannerBase {
* @deprecated since 1.4, use {@link #addScanIterator(IteratorSetting)}
* @see org.apache.accumulo.core.iterators.user.RegExFilter
*/
+ @Deprecated
@Override
public synchronized void setRowRegex(String regex) {
ArgumentChecker.notNull(regex);
@@ -180,6 +184,7 @@ public class ScannerOptions implements ScannerBase {
* @deprecated since 1.4, use{@link #addScanIterator(IteratorSetting)}
* @see org.apache.accumulo.core.iterators.user.RegExFilter
*/
+ @Deprecated
@Override
public synchronized void setColumnFamilyRegex(String regex) {
ArgumentChecker.notNull(regex);
@@ -194,6 +199,7 @@ public class ScannerOptions implements ScannerBase {
* @deprecated since 1.4, use {@link #addScanIterator(IteratorSetting)}
* @see org.apache.accumulo.core.iterators.user.RegExFilter
*/
+ @Deprecated
@Override
public synchronized void setColumnQualifierRegex(String regex) {
ArgumentChecker.notNull(regex);
@@ -208,6 +214,7 @@ public class ScannerOptions implements ScannerBase {
* @deprecated since 1.4, use {@link #addScanIterator(IteratorSetting)}
* @see org.apache.accumulo.core.iterators.user.RegExFilter
*/
+ @Deprecated
@Override
public synchronized void setValueRegex(String regex) {
ArgumentChecker.notNull(regex);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchDeleter.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchDeleter.java b/src/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchDeleter.java
index d9198bc..f727e0c 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchDeleter.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchDeleter.java
@@ -34,14 +34,14 @@ import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.accumulo.core.security.thrift.AuthInfo;
public class TabletServerBatchDeleter extends TabletServerBatchReader implements BatchDeleter {
-
+
private Instance instance;
private AuthInfo credentials;
private String tableId;
private long maxMemory;
private long maxLatency;
private int maxWriteThreads;
-
+
public TabletServerBatchDeleter(Instance instance, AuthInfo credentials, String tableId, Authorizations authorizations, int numQueryThreads, long maxMemory,
long maxLatency, int maxWriteThreads) throws TableNotFoundException {
super(instance, credentials, tableId, authorizations, numQueryThreads);
@@ -53,12 +53,13 @@ public class TabletServerBatchDeleter extends TabletServerBatchReader implements
this.maxWriteThreads = maxWriteThreads;
super.addScanIterator(new IteratorSetting(Integer.MAX_VALUE, BatchDeleter.class.getName() + ".NOVALUE", SortedKeyIterator.class));
}
-
+
+ @Deprecated
@Override
public synchronized void setValueRegex(String regex) {
throw new UnsupportedOperationException("Cannot filter on value with deleter; Write your own deleter");
}
-
+
@Override
public void delete() throws MutationsRejectedException, TableNotFoundException {
BatchWriter bw = null;
@@ -77,5 +78,5 @@ public class TabletServerBatchDeleter extends TabletServerBatchReader implements
bw.close();
}
}
-
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormat.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormat.java b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormat.java
index 67d220c..7cfab8b 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormat.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormat.java
@@ -90,6 +90,7 @@ public class AccumuloFileOutputFormat extends FileOutputFormat<Key,Value> {
/**
* @deprecated Use {@link #handleBlockSize(Configuration)} instead
*/
+ @Deprecated
protected static void handleBlockSize(JobContext job) {
handleBlockSize(job.getConfiguration());
}
@@ -108,6 +109,7 @@ public class AccumuloFileOutputFormat extends FileOutputFormat<Key,Value> {
/**
* @deprecated Use {@link #setFileType(Configuration,String)} instead
*/
+ @Deprecated
public static void setFileType(JobContext job, String type) {
setFileType(job.getConfiguration(), type);
}
@@ -119,6 +121,7 @@ public class AccumuloFileOutputFormat extends FileOutputFormat<Key,Value> {
/**
* @deprecated Use {@link #setBlockSize(Configuration,int)} instead
*/
+ @Deprecated
public static void setBlockSize(JobContext job, int blockSize) {
setBlockSize(job.getConfiguration(), blockSize);
}
@@ -132,22 +135,13 @@ public class AccumuloFileOutputFormat extends FileOutputFormat<Key,Value> {
}
/**
- *
- * @param job
- * @param instanceName
- * @param zooKeepers
* @deprecated since 1.4, use {@link #setZooKeeperInstance(Configuration conf, String instanceName, String zooKeepers)}
*/
+ @Deprecated
public static void setZooKeeperInstance(JobContext job, String instanceName, String zooKeepers) {
setZooKeeperInstance(job.getConfiguration(), instanceName, zooKeepers);
}
- /**
- *
- * @param conf
- * @param instanceName
- * @param zooKeepers
- */
public static void setZooKeeperInstance(Configuration conf, String instanceName, String zooKeepers) {
if (conf.getBoolean(INSTANCE_HAS_BEEN_SET, false))
throw new IllegalStateException("Instance info can only be set once per job");
@@ -159,19 +153,16 @@ public class AccumuloFileOutputFormat extends FileOutputFormat<Key,Value> {
}
/**
- *
- * @param job
* @return The Accumulo instance.
*
* @deprecated since 1.4, use {@link #getInstance(Configuration conf)} no other configurations are used by OutputFormat
*/
+ @Deprecated
protected static Instance getInstance(JobContext job) {
return getInstance(job.getConfiguration());
}
/**
- *
- * @param conf
* @return The Accumulo instance.
*/
protected static Instance getInstance(Configuration conf) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
index 02516a3..ed0aebf 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
@@ -103,6 +103,7 @@ public class AccumuloOutputFormat extends OutputFormat<Text,Mutation> {
* the table to use when the tablename is null in the write call
* @deprecated Use {@link #setOutputInfo(Configuration,String,byte[],boolean,String)} instead
*/
+ @Deprecated
public static void setOutputInfo(JobContext job, String user, byte[] passwd, boolean createTables, String defaultTable) {
setOutputInfo(job.getConfiguration(), user, passwd, createTables, defaultTable);
}
@@ -137,6 +138,7 @@ public class AccumuloOutputFormat extends OutputFormat<Text,Mutation> {
/**
* @deprecated Use {@link #setZooKeeperInstance(Configuration,String,String)} instead
*/
+ @Deprecated
public static void setZooKeeperInstance(JobContext job, String instanceName, String zooKeepers) {
setZooKeeperInstance(job.getConfiguration(), instanceName, zooKeepers);
}
@@ -154,6 +156,7 @@ public class AccumuloOutputFormat extends OutputFormat<Text,Mutation> {
/**
* @deprecated Use {@link #setMockInstance(Configuration,String)} instead
*/
+ @Deprecated
public static void setMockInstance(JobContext job, String instanceName) {
setMockInstance(job.getConfiguration(), instanceName);
}
@@ -167,6 +170,7 @@ public class AccumuloOutputFormat extends OutputFormat<Text,Mutation> {
/**
* @deprecated Use {@link #setMaxMutationBufferSize(Configuration,long)} instead
*/
+ @Deprecated
public static void setMaxMutationBufferSize(JobContext job, long numberOfBytes) {
setMaxMutationBufferSize(job.getConfiguration(), numberOfBytes);
}
@@ -178,6 +182,7 @@ public class AccumuloOutputFormat extends OutputFormat<Text,Mutation> {
/**
* @deprecated Use {@link #setMaxLatency(Configuration,int)} instead
*/
+ @Deprecated
public static void setMaxLatency(JobContext job, int numberOfMilliseconds) {
setMaxLatency(job.getConfiguration(), numberOfMilliseconds);
}
@@ -189,6 +194,7 @@ public class AccumuloOutputFormat extends OutputFormat<Text,Mutation> {
/**
* @deprecated Use {@link #setMaxWriteThreads(Configuration,int)} instead
*/
+ @Deprecated
public static void setMaxWriteThreads(JobContext job, int numberOfThreads) {
setMaxWriteThreads(job.getConfiguration(), numberOfThreads);
}
@@ -200,6 +206,7 @@ public class AccumuloOutputFormat extends OutputFormat<Text,Mutation> {
/**
* @deprecated Use {@link #setLogLevel(Configuration,Level)} instead
*/
+ @Deprecated
public static void setLogLevel(JobContext job, Level level) {
setLogLevel(job.getConfiguration(), level);
}
@@ -212,6 +219,7 @@ public class AccumuloOutputFormat extends OutputFormat<Text,Mutation> {
/**
* @deprecated Use {@link #setSimulationMode(Configuration)} instead
*/
+ @Deprecated
public static void setSimulationMode(JobContext job) {
setSimulationMode(job.getConfiguration());
}
@@ -223,6 +231,7 @@ public class AccumuloOutputFormat extends OutputFormat<Text,Mutation> {
/**
* @deprecated Use {@link #getUsername(Configuration)} instead
*/
+ @Deprecated
protected static String getUsername(JobContext job) {
return getUsername(job.getConfiguration());
}
@@ -237,6 +246,7 @@ public class AccumuloOutputFormat extends OutputFormat<Text,Mutation> {
*
* @deprecated Use {@link #getPassword(Configuration)} instead
*/
+ @Deprecated
protected static byte[] getPassword(JobContext job) {
return getPassword(job.getConfiguration());
}
@@ -252,6 +262,7 @@ public class AccumuloOutputFormat extends OutputFormat<Text,Mutation> {
/**
* @deprecated Use {@link #canCreateTables(Configuration)} instead
*/
+ @Deprecated
protected static boolean canCreateTables(JobContext job) {
return canCreateTables(job.getConfiguration());
}
@@ -263,6 +274,7 @@ public class AccumuloOutputFormat extends OutputFormat<Text,Mutation> {
/**
* @deprecated Use {@link #getDefaultTableName(Configuration)} instead
*/
+ @Deprecated
protected static String getDefaultTableName(JobContext job) {
return getDefaultTableName(job.getConfiguration());
}
@@ -274,6 +286,7 @@ public class AccumuloOutputFormat extends OutputFormat<Text,Mutation> {
/**
* @deprecated Use {@link #getInstance(Configuration)} instead
*/
+ @Deprecated
protected static Instance getInstance(JobContext job) {
return getInstance(job.getConfiguration());
}
@@ -287,6 +300,7 @@ public class AccumuloOutputFormat extends OutputFormat<Text,Mutation> {
/**
* @deprecated Use {@link #getMaxMutationBufferSize(Configuration)} instead
*/
+ @Deprecated
protected static long getMaxMutationBufferSize(JobContext job) {
return getMaxMutationBufferSize(job.getConfiguration());
}
@@ -298,6 +312,7 @@ public class AccumuloOutputFormat extends OutputFormat<Text,Mutation> {
/**
* @deprecated Use {@link #getMaxLatency(Configuration)} instead
*/
+ @Deprecated
protected static int getMaxLatency(JobContext job) {
return getMaxLatency(job.getConfiguration());
}
@@ -309,6 +324,7 @@ public class AccumuloOutputFormat extends OutputFormat<Text,Mutation> {
/**
* @deprecated Use {@link #getMaxWriteThreads(Configuration)} instead
*/
+ @Deprecated
protected static int getMaxWriteThreads(JobContext job) {
return getMaxWriteThreads(job.getConfiguration());
}
@@ -320,6 +336,7 @@ public class AccumuloOutputFormat extends OutputFormat<Text,Mutation> {
/**
* @deprecated Use {@link #getLogLevel(Configuration)} instead
*/
+ @Deprecated
protected static Level getLogLevel(JobContext job) {
return getLogLevel(job.getConfiguration());
}
@@ -333,6 +350,7 @@ public class AccumuloOutputFormat extends OutputFormat<Text,Mutation> {
/**
* @deprecated Use {@link #getSimulationMode(Configuration)} instead
*/
+ @Deprecated
protected static boolean getSimulationMode(JobContext job) {
return getSimulationMode(job.getConfiguration());
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
index 8fe0347..de97f12 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
@@ -146,6 +146,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #setIsolated(Configuration,boolean)} instead
*/
+ @Deprecated
public static void setIsolated(JobContext job, boolean enable) {
setIsolated(job.getConfiguration(), enable);
}
@@ -165,6 +166,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #setLocalIterators(Configuration,boolean)} instead
*/
+ @Deprecated
public static void setLocalIterators(JobContext job, boolean enable) {
setLocalIterators(job.getConfiguration(), enable);
}
@@ -184,6 +186,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #setInputInfo(Configuration,String,byte[],String,Authorizations)} instead
*/
+ @Deprecated
public static void setInputInfo(JobContext job, String user, byte[] passwd, String table, Authorizations auths) {
setInputInfo(job.getConfiguration(), user, passwd, table, auths);
}
@@ -218,6 +221,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #setZooKeeperInstance(Configuration,String,String)} instead
*/
+ @Deprecated
public static void setZooKeeperInstance(JobContext job, String instanceName, String zooKeepers) {
setZooKeeperInstance(job.getConfiguration(), instanceName, zooKeepers);
}
@@ -245,6 +249,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #setMockInstance(Configuration,String)} instead
*/
+ @Deprecated
public static void setMockInstance(JobContext job, String instanceName) {
setMockInstance(job.getConfiguration(), instanceName);
}
@@ -266,6 +271,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #setRanges(Configuration,Collection)} instead
*/
+ @Deprecated
public static void setRanges(JobContext job, Collection<Range> ranges) {
setRanges(job.getConfiguration(), ranges);
}
@@ -296,6 +302,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #disableAutoAdjustRanges(Configuration)} instead
*/
+ @Deprecated
public static void disableAutoAdjustRanges(JobContext job) {
disableAutoAdjustRanges(job.getConfiguration());
}
@@ -314,6 +321,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated since 1.4 use {@link org.apache.accumulo.core.iterators.user.RegExFilter} and {@link #addIterator(Configuration, IteratorSetting)}
*/
+ @Deprecated
public static enum RegexType {
ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, VALUE
}
@@ -325,6 +333,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
* @param type
* @param regex
*/
+ @Deprecated
public static void setRegex(JobContext job, RegexType type, String regex) {
ArgumentChecker.notNull(type, regex);
String key = null;
@@ -355,6 +364,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #setMaxVersions(Configuration,int)} instead
*/
+ @Deprecated
public static void setMaxVersions(JobContext job, int maxVersions) throws IOException {
setMaxVersions(job.getConfiguration(), maxVersions);
}
@@ -410,6 +420,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #fetchColumns(Configuration,Collection)} instead
*/
+ @Deprecated
public static void fetchColumns(JobContext job, Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
fetchColumns(job.getConfiguration(), columnFamilyColumnQualifierPairs);
}
@@ -447,6 +458,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #setLogLevel(Configuration,Level)} instead
*/
+ @Deprecated
public static void setLogLevel(JobContext job, Level level) {
setLogLevel(job.getConfiguration(), level);
}
@@ -468,6 +480,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #addIterator(Configuration,IteratorSetting)} instead
*/
+ @Deprecated
public static void addIterator(JobContext job, IteratorSetting cfg) {
addIterator(job.getConfiguration(), cfg);
}
@@ -527,6 +540,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
*
* @deprecated since 1.4, see {@link #addIterator(Configuration, IteratorSetting)}
*/
+ @Deprecated
public static void setIterator(JobContext job, int priority, String iteratorClass, String iteratorName) {
// First check to see if anything has been set already
String iterators = job.getConfiguration().get(ITERATORS);
@@ -557,6 +571,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
*
* @deprecated since 1.4, see {@link #addIterator(Configuration, IteratorSetting)}
*/
+ @Deprecated
public static void setIteratorOption(JobContext job, String iteratorName, String key, String value) {
if (iteratorName == null || key == null || value == null)
return;
@@ -578,6 +593,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #isIsolated(Configuration)} instead
*/
+ @Deprecated
protected static boolean isIsolated(JobContext job) {
return isIsolated(job.getConfiguration());
}
@@ -597,6 +613,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #usesLocalIterators(Configuration)} instead
*/
+ @Deprecated
protected static boolean usesLocalIterators(JobContext job) {
return usesLocalIterators(job.getConfiguration());
}
@@ -616,6 +633,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #getUsername(Configuration)} instead
*/
+ @Deprecated
protected static String getUsername(JobContext job) {
return getUsername(job.getConfiguration());
}
@@ -638,6 +656,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
*
* @deprecated Use {@link #getPassword(Configuration)} instead
*/
+ @Deprecated
protected static byte[] getPassword(JobContext job) {
return getPassword(job.getConfiguration());
}
@@ -658,6 +677,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #getTablename(Configuration)} instead
*/
+ @Deprecated
protected static String getTablename(JobContext job) {
return getTablename(job.getConfiguration());
}
@@ -677,6 +697,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #getAuthorizations(Configuration)} instead
*/
+ @Deprecated
protected static Authorizations getAuthorizations(JobContext job) {
return getAuthorizations(job.getConfiguration());
}
@@ -697,6 +718,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #getInstance(Configuration)} instead
*/
+ @Deprecated
protected static Instance getInstance(JobContext job) {
return getInstance(job.getConfiguration());
}
@@ -719,6 +741,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #getTabletLocator(Configuration)} instead
*/
+ @Deprecated
protected static TabletLocator getTabletLocator(JobContext job) throws TableNotFoundException {
return getTabletLocator(job.getConfiguration());
}
@@ -746,6 +769,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #getRanges(Configuration)} instead
*/
+ @Deprecated
protected static List<Range> getRanges(JobContext job) throws IOException {
return getRanges(job.getConfiguration());
}
@@ -775,6 +799,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
* @deprecated since 1.4 use {@link org.apache.accumulo.core.iterators.user.RegExFilter} and {@link #addIterator(Configuration, IteratorSetting)}
* @see #setRegex(JobContext, RegexType, String)
*/
+ @Deprecated
protected static String getRegex(JobContext job, RegexType type) {
String key = null;
switch (type) {
@@ -807,6 +832,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #getFetchedColumns(Configuration)} instead
*/
+ @Deprecated
protected static Set<Pair<Text,Text>> getFetchedColumns(JobContext job) {
return getFetchedColumns(job.getConfiguration());
}
@@ -844,6 +870,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #getAutoAdjustRanges(Configuration)} instead
*/
+ @Deprecated
protected static boolean getAutoAdjustRanges(JobContext job) {
return getAutoAdjustRanges(job.getConfiguration());
}
@@ -863,6 +890,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #getLogLevel(Configuration)} instead
*/
+ @Deprecated
protected static Level getLogLevel(JobContext job) {
return getLogLevel(job.getConfiguration());
}
@@ -884,6 +912,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #validateOptions(Configuration)} instead
*/
+ @Deprecated
protected static void validateOptions(JobContext job) throws IOException {
validateOptions(job.getConfiguration());
}
@@ -929,6 +958,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #getMaxVersions(Configuration)} instead
*/
+ @Deprecated
protected static int getMaxVersions(JobContext job) {
return getMaxVersions(job.getConfiguration());
}
@@ -954,6 +984,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #getIterators(Configuration)} instead
*/
+ @Deprecated
protected static List<AccumuloIterator> getIterators(JobContext job) {
return getIterators(job.getConfiguration());
}
@@ -987,6 +1018,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated Use {@link #getIteratorOptions(Configuration)} instead
*/
+ @Deprecated
protected static List<AccumuloIteratorOption> getIteratorOptions(JobContext job) {
return getIteratorOptions(job.getConfiguration());
}
@@ -1025,6 +1057,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated since 1.4, configure {@link org.apache.accumulo.core.iterators.user.RegExFilter} instead.
*/
+ @Deprecated
private void checkAndEnableRegex(String regex, Scanner scanner, String methodName) throws IllegalArgumentException, SecurityException,
IllegalAccessException, InvocationTargetException, NoSuchMethodException, IOException {
if (regex != null) {
@@ -1040,6 +1073,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
/**
* @deprecated since 1.4, configure {@link org.apache.accumulo.core.iterators.user.RegExFilter} instead.
*/
+ @Deprecated
protected boolean setupRegex(TaskAttemptContext attempt, Scanner scanner) throws AccumuloException {
try {
checkAndEnableRegex(getRegex(attempt, RegexType.ROW), scanner, "setRowRegex");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockAccumulo.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockAccumulo.java b/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockAccumulo.java
index c2d7d35..4b03038 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockAccumulo.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockAccumulo.java
@@ -77,6 +77,7 @@ public class MockAccumulo {
/**
* @deprecated since 1.4
*/
+ @Deprecated
public void addAggregators(String tableName, List<? extends PerColumnIteratorConfig> aggregators) {
tables.get(tableName).addAggregators(aggregators);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java b/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java
index 999c9b2..75a96c5 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java
@@ -119,6 +119,7 @@ public class MockTable {
/**
* @deprecated Since 1.4, use {@link TableOperations#attachIterator(String tableName, IteratorSetting setting)}
*/
+ @Deprecated
public void addAggregators(List<? extends org.apache.accumulo.core.iterators.conf.PerColumnIteratorConfig> aggregators) {
for (Entry<String,String> entry : IteratorUtil.generateAggTableProperties(aggregators).entrySet()) {
String key = entry.getKey();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java b/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java
index a5ef427..8e157a1 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java
@@ -102,6 +102,7 @@ public class MockTableOperations extends TableOperationsHelper {
/**
* @deprecated since 1.4 {@link #attachIterator(String, IteratorSetting)}
*/
+ @Deprecated
@Override
public void addAggregators(String tableName, List<? extends PerColumnIteratorConfig> aggregators) throws AccumuloSecurityException, TableNotFoundException,
AccumuloException {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java b/src/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java
index 07c8864..ae123bc 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java
@@ -137,6 +137,7 @@ public abstract class AccumuloConfiguration implements Iterable<Entry<String,Str
*
* @deprecated not for client use
*/
+ @Deprecated
public static synchronized AccumuloConfiguration getSiteConfiguration() {
return SiteConfiguration.getInstance(getDefaultConfiguration());
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/data/Value.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/data/Value.java b/src/core/src/main/java/org/apache/accumulo/core/data/Value.java
index 683d23f..5cbe748 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/data/Value.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/data/Value.java
@@ -251,6 +251,7 @@ public class Value implements WritableComparable<Object> {
*
* @deprecated since 1.4.1. There is no replacement.
*/
+ @Deprecated
public static byte[] longToBytes(long val) throws IOException {
return getBytes(new LongWritable(val));
}
@@ -260,6 +261,7 @@ public class Value implements WritableComparable<Object> {
*
* @deprecated since 1.4.1. There is no replacement.
*/
+ @Deprecated
public static long bytesToLong(byte[] bytes) throws IOException {
if (bytes == null || bytes.length == 0) {
return -1L;
@@ -272,6 +274,7 @@ public class Value implements WritableComparable<Object> {
*
* @deprecated since 1.4.1. There is no replacement.
*/
+ @Deprecated
public static byte[] stringToBytes(String s) throws UnsupportedEncodingException {
if (s == null) {
throw new IllegalArgumentException("string cannot be null");
@@ -284,6 +287,7 @@ public class Value implements WritableComparable<Object> {
*
* @deprecated since 1.4.1. There is no replacement.
*/
+ @Deprecated
public static String bytesToString(byte[] bytes) throws UnsupportedEncodingException {
if (bytes == null || bytes.length == 0) {
return "";
@@ -297,6 +301,7 @@ public class Value implements WritableComparable<Object> {
*
* @deprecated since 1.4.1. There is no replacement.
*/
+ @Deprecated
public static byte[] getBytes(final Writable w) throws IOException {
if (w == null) {
throw new IllegalArgumentException("Writable cannot be null");
@@ -325,6 +330,7 @@ public class Value implements WritableComparable<Object> {
*
* @deprecated since 1.4.1. There is no replacement.
*/
+ @Deprecated
public static Writable getWritable(final byte[] bytes, final Writable w) throws IOException {
if (bytes == null || bytes.length == 0) {
throw new IllegalArgumentException("Can't build a writable with empty " + "bytes array");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java b/src/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
index 1df0491..f580a12 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
@@ -67,12 +67,13 @@ public class BloomFilterLayer {
private static final Logger LOG = Logger.getLogger(BloomFilterLayer.class);
public static final String BLOOM_FILE_NAME = "acu_bloom";
public static final int HASH_COUNT = 5;
-
+
private static class BloomLoaderThreadFactory implements ThreadFactory {
-
+
private ThreadFactory dtf = Executors.defaultThreadFactory();
private int threadNum = 1;
-
+
+ @Override
public Thread newThread(Runnable r) {
Thread thread = dtf.newThread(r);
thread.setName("bloom-loader-" + threadNum++);
@@ -80,37 +81,37 @@ public class BloomFilterLayer {
return thread;
}
}
-
+
private static ExecutorService loadThreadPool = null;
-
+
private static synchronized ExecutorService getLoadThreadPool(int maxLoadThreads) {
if (loadThreadPool != null) {
return loadThreadPool;
}
-
+
if (maxLoadThreads > 0) {
loadThreadPool = new ThreadPoolExecutor(0, maxLoadThreads, 60, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new BloomLoaderThreadFactory());
}
-
+
return loadThreadPool;
}
-
+
public static class Writer implements FileSKVWriter {
private DynamicBloomFilter bloomFilter;
private int numKeys;
private int vectorSize;
-
+
private FileSKVWriter writer;
private KeyFunctor transformer = null;
private boolean closed = false;
-
+
Writer(FileSKVWriter writer, AccumuloConfiguration acuconf) {
this.writer = writer;
initBloomFilter(acuconf);
}
-
+
private synchronized void initBloomFilter(AccumuloConfiguration acuconf) {
-
+
numKeys = acuconf.getCount(Property.TABLE_BLOOM_SIZE);
// vector size should be <code>-kn / (ln(1 - c^(1/k)))</code> bits for
// single key, where <code> is the number of hash functions,
@@ -120,22 +121,22 @@ public class BloomFilterLayer {
double errorRate = acuconf.getFraction(Property.TABLE_BLOOM_ERRORRATE);
vectorSize = (int) Math.ceil(-HASH_COUNT * numKeys / Math.log(1.0 - Math.pow(errorRate, 1.0 / HASH_COUNT)));
bloomFilter = new DynamicBloomFilter(vectorSize, HASH_COUNT, Hash.parseHashType(acuconf.get(Property.TABLE_BLOOM_HASHTYPE)), numKeys);
-
+
/**
* load KeyFunctor
*/
try {
Class<? extends KeyFunctor> clazz = AccumuloClassLoader.loadClass(acuconf.get(Property.TABLE_BLOOM_KEY_FUNCTOR), KeyFunctor.class);
transformer = clazz.newInstance();
-
+
} catch (Exception e) {
LOG.error("Failed to find KeyFunctor: " + acuconf.get(Property.TABLE_BLOOM_KEY_FUNCTOR), e);
throw new IllegalArgumentException("Failed to find KeyFunctor: " + acuconf.get(Property.TABLE_BLOOM_KEY_FUNCTOR));
-
+
}
-
+
}
-
+
@Override
public synchronized void append(org.apache.accumulo.core.data.Key key, Value val) throws IOException {
writer.append(key, val);
@@ -143,13 +144,13 @@ public class BloomFilterLayer {
if (bloomKey.getBytes().length > 0)
bloomFilter.add(bloomKey);
}
-
+
@Override
public synchronized void close() throws IOException {
-
+
if (closed)
return;
-
+
DataOutputStream out = writer.createMetaStore(BLOOM_FILE_NAME);
out.writeUTF(transformer.getClass().getCanonicalName());
bloomFilter.write(out);
@@ -158,31 +159,31 @@ public class BloomFilterLayer {
writer.close();
closed = true;
}
-
+
@Override
public DataOutputStream createMetaStore(String name) throws IOException {
return writer.createMetaStore(name);
}
-
+
@Override
public void startDefaultLocalityGroup() throws IOException {
writer.startDefaultLocalityGroup();
-
+
}
-
+
@Override
public void startNewLocalityGroup(String name, Set<ByteSequence> columnFamilies) throws IOException {
writer.startNewLocalityGroup(name, columnFamilies);
}
-
+
@Override
public boolean supportsLocalityGroups() {
return writer.supportsLocalityGroups();
}
}
-
+
static class BloomFilterLoader {
-
+
private volatile DynamicBloomFilter bloomFilter;
private int loadRequest = 0;
private int loadThreshold = 1;
@@ -190,42 +191,43 @@ public class BloomFilterLayer {
private Runnable loadTask;
private volatile KeyFunctor transformer = null;
private volatile boolean closed = false;
-
+
BloomFilterLoader(final FileSKVIterator reader, AccumuloConfiguration acuconf) {
-
+
maxLoadThreads = acuconf.getCount(Property.TSERV_BLOOM_LOAD_MAXCONCURRENT);
-
+
loadThreshold = acuconf.getCount(Property.TABLE_BLOOM_LOAD_THRESHOLD);
-
+
loadTask = new Runnable() {
+ @Override
public void run() {
-
+
// no need to load the bloom filter if the map file is closed
if (closed)
return;
String ClassName = null;
DataInputStream in = null;
-
+
try {
in = reader.getMetaStore(BLOOM_FILE_NAME);
DynamicBloomFilter tmpBloomFilter = new DynamicBloomFilter();
-
+
// check for closed again after open but before reading the bloom filter in
if (closed)
return;
-
+
/**
* Load classname for keyFunctor
*/
ClassName = in.readUTF();
-
+
Class<? extends KeyFunctor> clazz = AccumuloClassLoader.loadClass(ClassName, KeyFunctor.class);
transformer = clazz.newInstance();
-
+
/**
* read in bloom filter
*/
-
+
tmpBloomFilter.readFields(in);
// only set the bloom filter after it is fully constructed
bloomFilter = tmpBloomFilter;
@@ -236,7 +238,7 @@ public class BloomFilterLayer {
LOG.warn("Can't open BloomFilter", ioe);
else
LOG.debug("Can't open BloomFilter, file closed : " + ioe.getMessage());
-
+
bloomFilter = null;
} catch (ClassNotFoundException e) {
LOG.error("Failed to find KeyFunctor in config: " + ClassName, e);
@@ -252,7 +254,7 @@ public class BloomFilterLayer {
throw npe;
else
LOG.debug("Can't open BloomFilter, NPE after closed ", npe);
-
+
} finally {
if (in != null) {
try {
@@ -264,11 +266,11 @@ public class BloomFilterLayer {
}
}
};
-
+
initiateLoad(maxLoadThreads);
-
+
}
-
+
private synchronized void initiateLoad(int maxLoadThreads) {
// ensure only one thread initiates loading of bloom filter by
// only taking action when loadTask != null
@@ -287,16 +289,14 @@ public class BloomFilterLayer {
loadTask = null;
}
}
-
+
loadRequest++;
}
-
+
/**
* Checks if this MyMapFile has the indicated key. The membership test is performed using a Bloom filter, so the result has always non-zero probability of
* false positives.
*
- * @param key
- * key to check
* @return false iff key doesn't exist, true if key probably exists.
* @throws IOException
*/
@@ -306,45 +306,45 @@ public class BloomFilterLayer {
if (bloomFilter == null)
return true;
}
-
+
Key bloomKey = transformer.transform(range);
-
+
if (bloomKey == null || bloomKey.getBytes().length == 0)
return true;
-
+
return bloomFilter.membershipTest(bloomKey);
}
-
+
public void close() {
this.closed = true;
}
}
-
+
public static class Reader implements FileSKVIterator {
-
+
private BloomFilterLoader bfl;
private FileSKVIterator reader;
-
+
public Reader(FileSKVIterator reader, AccumuloConfiguration acuconf) {
this.reader = reader;
bfl = new BloomFilterLoader(reader, acuconf);
}
-
+
private Reader(FileSKVIterator src, BloomFilterLoader bfl) {
this.reader = src;
this.bfl = bfl;
}
-
+
private boolean checkSuper = true;
-
+
@Override
public boolean hasTop() {
return checkSuper ? reader.hasTop() : false;
}
-
+
@Override
public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
-
+
if (!bfl.probablyHasKey(range)) {
checkSuper = false;
} else {
@@ -352,117 +352,118 @@ public class BloomFilterLayer {
checkSuper = true;
}
}
-
+
+ @Override
public synchronized void close() throws IOException {
bfl.close();
reader.close();
}
-
+
@Override
public org.apache.accumulo.core.data.Key getFirstKey() throws IOException {
return reader.getFirstKey();
}
-
+
@Override
public org.apache.accumulo.core.data.Key getLastKey() throws IOException {
return reader.getLastKey();
}
-
+
@Override
public SortedKeyValueIterator<org.apache.accumulo.core.data.Key,Value> deepCopy(IteratorEnvironment env) {
return new BloomFilterLayer.Reader((FileSKVIterator) reader.deepCopy(env), bfl);
}
-
+
@Override
public org.apache.accumulo.core.data.Key getTopKey() {
return reader.getTopKey();
}
-
+
@Override
public Value getTopValue() {
return reader.getTopValue();
}
-
+
@Override
public void init(SortedKeyValueIterator<org.apache.accumulo.core.data.Key,Value> source, Map<String,String> options, IteratorEnvironment env)
throws IOException {
throw new UnsupportedOperationException();
-
+
}
-
+
@Override
public void next() throws IOException {
reader.next();
}
-
+
@Override
public DataInputStream getMetaStore(String name) throws IOException {
return reader.getMetaStore(name);
}
-
+
@Override
public void closeDeepCopies() throws IOException {
reader.closeDeepCopies();
}
-
+
@Override
public void setInterruptFlag(AtomicBoolean flag) {
reader.setInterruptFlag(flag);
}
-
+
}
-
+
public static void main(String[] args) throws IOException {
PrintStream out = System.out;
-
+
Random r = new Random();
-
+
HashSet<Integer> valsSet = new HashSet<Integer>();
-
+
for (int i = 0; i < 100000; i++) {
valsSet.add(r.nextInt(Integer.MAX_VALUE));
}
-
+
ArrayList<Integer> vals = new ArrayList<Integer>(valsSet);
Collections.sort(vals);
-
+
ConfigurationCopy acuconf = new ConfigurationCopy(AccumuloConfiguration.getDefaultConfiguration());
acuconf.set(Property.TABLE_BLOOM_ENABLED, "true");
acuconf.set(Property.TABLE_BLOOM_KEY_FUNCTOR, "accumulo.core.file.keyfunctor.ColumnFamilyFunctor");
acuconf.set(Property.TABLE_FILE_TYPE, RFile.EXTENSION);
acuconf.set(Property.TABLE_BLOOM_LOAD_THRESHOLD, "1");
acuconf.set(Property.TSERV_BLOOM_LOAD_MAXCONCURRENT, "1");
-
+
Configuration conf = CachedConfiguration.getInstance();
FileSystem fs = FileSystem.get(conf);
-
+
String suffix = FileOperations.getNewFileExtension(acuconf);
String fname = "/tmp/test." + suffix;
FileSKVWriter bmfw = FileOperations.getInstance().openWriter(fname, fs, conf, acuconf);
-
+
long t1 = System.currentTimeMillis();
-
+
bmfw.startDefaultLocalityGroup();
-
+
for (Integer i : vals) {
String fi = String.format("%010d", i);
bmfw.append(new org.apache.accumulo.core.data.Key(new Text("r" + fi), new Text("cf1")), new Value(("v" + fi).getBytes()));
bmfw.append(new org.apache.accumulo.core.data.Key(new Text("r" + fi), new Text("cf2")), new Value(("v" + fi).getBytes()));
}
-
+
long t2 = System.currentTimeMillis();
-
+
out.printf("write rate %6.2f\n", vals.size() / ((t2 - t1) / 1000.0));
-
+
bmfw.close();
-
+
t1 = System.currentTimeMillis();
FileSKVIterator bmfr = FileOperations.getInstance().openReader(fname, false, fs, conf, acuconf);
t2 = System.currentTimeMillis();
out.println("Opened " + fname + " in " + (t2 - t1));
-
+
t1 = System.currentTimeMillis();
-
+
int hits = 0;
for (int i = 0; i < 5000; i++) {
int row = r.nextInt(Integer.MAX_VALUE);
@@ -477,36 +478,36 @@ public class BloomFilterLayer {
}
}
}
-
+
t2 = System.currentTimeMillis();
-
+
out.printf("random lookup rate : %6.2f\n", 5000 / ((t2 - t1) / 1000.0));
out.println("hits = " + hits);
-
+
int count = 0;
-
+
t1 = System.currentTimeMillis();
-
+
for (Integer row : valsSet) {
String fi = String.format("%010d", row);
// bmfr.seek(new Range(new Text("r"+fi)));
-
+
org.apache.accumulo.core.data.Key k1 = new org.apache.accumulo.core.data.Key(new Text("r" + fi), new Text("cf1"));
bmfr.seek(new Range(k1, true, k1.followingKey(PartialKey.ROW_COLFAM), false), new ArrayList<ByteSequence>(), false);
-
+
if (!bmfr.hasTop()) {
out.println("ERROR 2 " + row);
}
-
+
count++;
-
+
if (count >= 500) {
break;
}
}
-
+
t2 = System.currentTimeMillis();
-
+
out.printf("existant lookup rate %6.2f\n", 500 / ((t2 - t1) / 1000.0));
out.println("expected hits 500. Receive hits: " + count);
bmfr.close();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/file/map/MapFileOperations.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/file/map/MapFileOperations.java b/src/core/src/main/java/org/apache/accumulo/core/file/map/MapFileOperations.java
index 4b6bc68..3890b1b 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/file/map/MapFileOperations.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/file/map/MapFileOperations.java
@@ -148,6 +148,7 @@ public class MapFileOperations extends FileOperations {
* @deprecated since 1.4, MapFiles will no longer be written. Instead use {@link org.apache.accumulo.core.file.rfile.RFileOperations#openWriter}.
* @see org.apache.hadoop.io.MapFile.Writer
*/
+ @Deprecated
public FileSKVWriter openWriter(final String file, final FileSystem fs, Configuration conf, AccumuloConfiguration acuconf) throws IOException {
final MyMapFile.Writer mfw = MapFileUtil.openMapFileWriter(acuconf, conf, fs, file);
return new FileSKVWriter() {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/file/map/MapFileUtil.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/file/map/MapFileUtil.java b/src/core/src/main/java/org/apache/accumulo/core/file/map/MapFileUtil.java
index abedb1a..f290222 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/file/map/MapFileUtil.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/file/map/MapFileUtil.java
@@ -37,6 +37,7 @@ public class MapFileUtil {
* @param dirName
* @return true if the attempt was successful, false otherwise
*/
+ @Deprecated
public static boolean attemptToFixMapFile(Configuration conf, FileSystem fs, String dirName) {
boolean fixed = true;
try {
@@ -99,6 +100,7 @@ public class MapFileUtil {
* @return a MapFile writer
* @throws IOException
*/
+ @Deprecated
public static MyMapFile.Writer openMapFileWriter(AccumuloConfiguration acuTableConf, Configuration conf, FileSystem fs, String dirname) throws IOException {
MyMapFile.Writer mfw = null;
int hbs = conf.getInt("io.seqfile.compress.blocksize", -1);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/file/map/MyMapFile.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/file/map/MyMapFile.java b/src/core/src/main/java/org/apache/accumulo/core/file/map/MyMapFile.java
index 780e99b..055e6fa 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/file/map/MyMapFile.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/file/map/MyMapFile.java
@@ -75,12 +75,14 @@ import org.apache.log4j.Logger;
*
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.file.rfile.RFile}
*/
+@Deprecated
@SuppressWarnings({"rawtypes", "unchecked"})
public class MyMapFile {
/**
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.Constants#MAPFILE_EXTENSION}
*/
+ @Deprecated
public static final String EXTENSION = "map";
private static final Logger log = Logger.getLogger(MyMapFile.class);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/file/map/MySequenceFile.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/file/map/MySequenceFile.java b/src/core/src/main/java/org/apache/accumulo/core/file/map/MySequenceFile.java
index 3b3596c..44a4095 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/file/map/MySequenceFile.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/file/map/MySequenceFile.java
@@ -194,6 +194,7 @@ import org.apache.hadoop.util.ReflectionUtils;
* @see CompressionCodec
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.file.rfile.RFile}
*/
+@Deprecated
@SuppressWarnings({"rawtypes", "unchecked"})
public class MySequenceFile {
private static final Log LOG = LogFactory.getLog(MySequenceFile.class);
[4/5] ACCUMULO-1961 Fix warnings
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/file/rfile/RelativeKey.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/file/rfile/RelativeKey.java b/src/core/src/main/java/org/apache/accumulo/core/file/rfile/RelativeKey.java
index 86aaa2f..b0c832a 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/file/rfile/RelativeKey.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/file/rfile/RelativeKey.java
@@ -34,296 +34,298 @@ import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableUtils;
public class RelativeKey implements WritableComparable<RelativeKey> {
-
+
private Key key;
-
+
private byte fieldsSame;
-
+
private Key prevKey;
-
+
private static final byte ROW_SAME = 0x01;
private static final byte CF_SAME = 0x02;
private static final byte CQ_SAME = 0x04;
private static final byte CV_SAME = 0x08;
private static final byte TS_SAME = 0x10;
private static final byte DELETED = 0x20;
-
+
private static HashMap<Text,Integer> colFams = new HashMap<Text,Integer>();
-
+
private static long bytesWritten = 0;
-
+
public static void printStats() throws Exception {
System.out.println("colFams.size() : " + colFams.size());
Set<Entry<Text,Integer>> es = colFams.entrySet();
-
+
int sum = 0;
-
+
for (Entry<Text,Integer> entry : es) {
sum += entry.getKey().getLength();
}
-
+
System.out.println("Total Column name bytes : " + sum);
-
+
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(new GZIPOutputStream(baos));
for (Entry<Text,Integer> entry : es) {
entry.getKey().write(dos);
dos.writeInt(entry.getValue());
}
-
+
dos.close();
-
+
System.out.println("Compressed column map size : " + baos.toByteArray().length);
System.out.printf("Bytes written : %,d\n", bytesWritten);
-
+
}
-
+
public RelativeKey() {
-
+
}
-
+
public RelativeKey(Key prevKey, Key key) {
-
+
this.key = key;
-
+
fieldsSame = 0;
-
+
if (prevKey != null) {
if (prevKey.getRowData().equals(key.getRowData()))
fieldsSame |= ROW_SAME;
-
+
if (prevKey.getColumnFamilyData().equals(key.getColumnFamilyData()))
fieldsSame |= CF_SAME;
-
+
if (prevKey.getColumnQualifierData().equals(key.getColumnQualifierData()))
fieldsSame |= CQ_SAME;
-
+
if (prevKey.getColumnVisibilityData().equals(key.getColumnVisibilityData()))
fieldsSame |= CV_SAME;
-
+
if (prevKey.getTimestamp() == key.getTimestamp())
fieldsSame |= TS_SAME;
-
+
}
-
+
// stored deleted information in bit vector instead of its own byte
if (key.isDeleted())
fieldsSame |= DELETED;
}
-
+
@Override
public void readFields(DataInput in) throws IOException {
fieldsSame = in.readByte();
-
+
byte[] row, cf, cq, cv;
long ts;
-
+
if ((fieldsSame & ROW_SAME) == 0) {
row = read(in);
} else {
row = prevKey.getRowData().toArray();
}
-
+
if ((fieldsSame & CF_SAME) == 0) {
cf = read(in);
} else {
cf = prevKey.getColumnFamilyData().toArray();
}
-
+
if ((fieldsSame & CQ_SAME) == 0) {
cq = read(in);
} else {
cq = prevKey.getColumnQualifierData().toArray();
}
-
+
if ((fieldsSame & CV_SAME) == 0) {
cv = read(in);
} else {
cv = prevKey.getColumnVisibilityData().toArray();
}
-
+
if ((fieldsSame & TS_SAME) == 0) {
ts = WritableUtils.readVLong(in);
} else {
ts = prevKey.getTimestamp();
}
-
+
this.key = new Key(row, cf, cq, cv, ts, (fieldsSame & DELETED) != 0, false);
-
+
this.prevKey = this.key;
}
-
+
static class MByteSequence extends ArrayByteSequence {
+ private static final long serialVersionUID = 1L;
+
MByteSequence(byte[] data, int offset, int length) {
super(data, offset, length);
}
-
+
MByteSequence(ByteSequence bs) {
super(new byte[Math.max(64, bs.length())]);
System.arraycopy(bs.getBackingArray(), bs.offset(), data, 0, bs.length());
this.length = bs.length();
this.offset = 0;
}
-
+
void setArray(byte[] data) {
this.data = data;
this.offset = 0;
this.length = 0;
}
-
+
void setLength(int len) {
this.length = len;
}
}
-
+
int fastSkip(DataInput in, Key seekKey, MByteSequence value, Key pkey, Key currKey) throws IOException {
// this method assumes that fast skip is being called on a compressed block where the last key
// in the compressed block is >= seekKey... therefore this method should go passed the end of the
// compressed block... if it does, there is probably an error in the callers logic
-
+
// this method mostly avoids object allocation and only does compares when the row changes
-
+
MByteSequence row, cf, cq, cv;
MByteSequence prow, pcf, pcq, pcv;
-
+
ByteSequence stopRow = seekKey.getRowData();
ByteSequence stopCF = seekKey.getColumnFamilyData();
ByteSequence stopCQ = seekKey.getColumnQualifierData();
-
+
long ts = -1;
long pts = -1;
boolean pdel = false;
-
+
int rowCmp = -1, cfCmp = -1, cqCmp = -1;
-
+
if (currKey != null) {
-
+
prow = new MByteSequence(pkey.getRowData());
pcf = new MByteSequence(pkey.getColumnFamilyData());
pcq = new MByteSequence(pkey.getColumnQualifierData());
pcv = new MByteSequence(pkey.getColumnVisibilityData());
pts = pkey.getTimestamp();
-
+
row = new MByteSequence(currKey.getRowData());
cf = new MByteSequence(currKey.getColumnFamilyData());
cq = new MByteSequence(currKey.getColumnQualifierData());
cv = new MByteSequence(currKey.getColumnVisibilityData());
ts = currKey.getTimestamp();
-
+
rowCmp = row.compareTo(stopRow);
cfCmp = cf.compareTo(stopCF);
cqCmp = cq.compareTo(stopCQ);
-
+
if (rowCmp >= 0) {
if (rowCmp > 0)
return 0;
-
+
if (cfCmp >= 0) {
if (cfCmp > 0)
return 0;
-
+
if (cqCmp >= 0)
return 0;
}
}
-
+
} else {
row = new MByteSequence(new byte[64], 0, 0);
cf = new MByteSequence(new byte[64], 0, 0);
cq = new MByteSequence(new byte[64], 0, 0);
cv = new MByteSequence(new byte[64], 0, 0);
-
+
prow = new MByteSequence(new byte[64], 0, 0);
pcf = new MByteSequence(new byte[64], 0, 0);
pcq = new MByteSequence(new byte[64], 0, 0);
pcv = new MByteSequence(new byte[64], 0, 0);
}
-
+
byte fieldsSame = -1;
int count = 0;
-
+
while (true) {
-
+
pdel = (fieldsSame & DELETED) != 0;
-
+
fieldsSame = in.readByte();
-
+
boolean changed = false;
-
+
if ((fieldsSame & ROW_SAME) == 0) {
-
+
MByteSequence tmp = prow;
prow = row;
row = tmp;
-
+
read(in, row);
-
+
// read a new row, so need to compare...
rowCmp = row.compareTo(stopRow);
changed = true;
}// else the row is the same as the last, so no need to compare
-
+
if ((fieldsSame & CF_SAME) == 0) {
-
+
MByteSequence tmp = pcf;
pcf = cf;
cf = tmp;
-
+
read(in, cf);
-
+
cfCmp = cf.compareTo(stopCF);
changed = true;
}
-
+
if ((fieldsSame & CQ_SAME) == 0) {
-
+
MByteSequence tmp = pcq;
pcq = cq;
cq = tmp;
-
+
read(in, cq);
-
+
cqCmp = cq.compareTo(stopCQ);
changed = true;
}
-
+
if ((fieldsSame & CV_SAME) == 0) {
-
+
MByteSequence tmp = pcv;
pcv = cv;
cv = tmp;
-
+
read(in, cv);
}
-
+
if ((fieldsSame & TS_SAME) == 0) {
pts = ts;
ts = WritableUtils.readVLong(in);
}
-
+
readValue(in, value);
-
+
count++;
-
+
if (changed && rowCmp >= 0) {
if (rowCmp > 0)
break;
-
+
if (cfCmp >= 0) {
if (cfCmp > 0)
break;
-
+
if (cqCmp >= 0)
break;
}
}
-
+
}
-
+
if (count > 1) {
MByteSequence trow, tcf, tcq, tcv;
long tts;
-
+
// when the current keys field is same as the last, then
// set the prev keys field the same as the current key
trow = (fieldsSame & ROW_SAME) == 0 ? prow : row;
@@ -331,104 +333,104 @@ public class RelativeKey implements WritableComparable<RelativeKey> {
tcq = (fieldsSame & CQ_SAME) == 0 ? pcq : cq;
tcv = (fieldsSame & CV_SAME) == 0 ? pcv : cv;
tts = (fieldsSame & TS_SAME) == 0 ? pts : ts;
-
+
Key tmp = new Key(trow.getBackingArray(), trow.offset(), trow.length(), tcf.getBackingArray(), tcf.offset(), tcf.length(), tcq.getBackingArray(),
tcq.offset(), tcq.length(), tcv.getBackingArray(), tcv.offset(), tcv.length(), tts);
tmp.setDeleted(pdel);
pkey.set(tmp);
}
-
+
this.key = new Key(row.getBackingArray(), row.offset(), row.length(), cf.getBackingArray(), cf.offset(), cf.length(), cq.getBackingArray(), cq.offset(),
cq.length(), cv.getBackingArray(), cv.offset(), cv.length(), ts);
this.key.setDeleted((fieldsSame & DELETED) != 0);
-
+
this.prevKey = this.key;
-
+
return count;
}
-
+
private void read(DataInput in, MByteSequence mbseq) throws IOException {
int len = WritableUtils.readVInt(in);
read(in, mbseq, len);
}
-
+
private void readValue(DataInput in, MByteSequence mbseq) throws IOException {
int len = in.readInt();
read(in, mbseq, len);
}
-
+
private void read(DataInput in, MByteSequence mbseq, int len) throws IOException {
if (mbseq.getBackingArray().length < len) {
int newLen = mbseq.getBackingArray().length;
-
+
while (newLen < len) {
newLen = newLen * 2;
}
-
+
mbseq.setArray(new byte[newLen]);
}
-
+
in.readFully(mbseq.getBackingArray(), 0, len);
mbseq.setLength(len);
}
-
+
private byte[] read(DataInput in) throws IOException {
int len = WritableUtils.readVInt(in);
byte[] data = new byte[len];
in.readFully(data);
return data;
}
-
+
public Key getKey() {
return key;
}
-
+
private void write(DataOutput out, ByteSequence bs) throws IOException {
WritableUtils.writeVInt(out, bs.length());
out.write(bs.getBackingArray(), bs.offset(), bs.length());
}
-
+
@Override
public void write(DataOutput out) throws IOException {
-
+
out.writeByte(fieldsSame);
-
+
// System.out.printf("wrote fs %x\n", fieldsSame);
-
+
bytesWritten += 1;
-
+
if ((fieldsSame & ROW_SAME) == 0) {
write(out, key.getRowData());
}
-
+
if ((fieldsSame & CF_SAME) == 0) {
write(out, key.getColumnFamilyData());
}
-
+
if ((fieldsSame & CQ_SAME) == 0) {
-
+
write(out, key.getColumnQualifierData());
-
+
/*
* Integer id = colFams.get(key.getColumnQualifier()); if(id == null){ id = nextId++; colFams.put(key.getColumnQualifier(), id); }
*
* WritableUtils.writeVInt(out, id); bytesWritten += 1;
*/
-
+
}
-
+
if ((fieldsSame & CV_SAME) == 0) {
write(out, key.getColumnVisibilityData());
}
-
+
if ((fieldsSame & TS_SAME) == 0) {
WritableUtils.writeVLong(out, key.getTimestamp());
}
}
-
+
@Override
public int compareTo(RelativeKey o) {
throw new UnsupportedOperationException();
}
-
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java b/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index fb7e335..5d0390d 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@ -158,9 +158,6 @@ public final class BCFile {
/**
* Current size of compressed data.
- *
- * @return
- * @throws IOException
*/
long getCompressedSize() throws IOException {
long ret = getCurrentPos() - posStart;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFile.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFile.java b/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFile.java
index f789f5a..412fd4c 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFile.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/TFile.java
@@ -103,27 +103,27 @@ import org.apache.hadoop.io.serializer.JavaSerializationComparator;
*/
public class TFile {
static final Log LOG = LogFactory.getLog(TFile.class);
-
+
private static final String CHUNK_BUF_SIZE_ATTR = "tfile.io.chunk.size";
private static final String FS_INPUT_BUF_SIZE_ATTR = "tfile.fs.input.buffer.size";
private static final String FS_OUTPUT_BUF_SIZE_ATTR = "tfile.fs.output.buffer.size";
-
+
static int getChunkBufferSize(Configuration conf) {
int ret = conf.getInt(CHUNK_BUF_SIZE_ATTR, 1024 * 1024);
return (ret > 0) ? ret : 1024 * 1024;
}
-
+
static int getFSInputBufferSize(Configuration conf) {
return conf.getInt(FS_INPUT_BUF_SIZE_ATTR, 256 * 1024);
}
-
+
static int getFSOutputBufferSize(Configuration conf) {
return conf.getInt(FS_OUTPUT_BUF_SIZE_ATTR, 256 * 1024);
}
-
+
private static final int MAX_KEY_SIZE = 64 * 1024; // 64KB
static final Version API_VERSION = new Version((short) 1, (short) 0);
-
+
/** snappy codec **/
public static final String COMPRESSION_SNAPPY = "snappy";
@@ -137,7 +137,7 @@ public class TFile {
public static final String COMPARATOR_MEMCMP = "memcmp";
/** comparator prefix: java class */
public static final String COMPARATOR_JCLASS = "jclass:";
-
+
/**
* Make a raw comparator from a string name.
*
@@ -148,12 +148,12 @@ public class TFile {
static public Comparator<RawComparable> makeComparator(String name) {
return TFileMeta.makeComparator(name);
}
-
+
// Prevent the instantiation of TFiles
private TFile() {
// nothing
}
-
+
/**
* Get names of supported compression algorithms. The names are acceptable by TFile.Writer.
*
@@ -168,32 +168,32 @@ public class TFile {
public static String[] getSupportedCompressionAlgorithms() {
return Compression.getSupportedAlgorithms();
}
-
+
/**
* TFile Writer.
*/
public static class Writer implements Closeable {
// minimum compressed size for a block.
private final int sizeMinBlock;
-
+
// Meta blocks.
final TFileIndex tfileIndex;
final TFileMeta tfileMeta;
-
+
// reference to the underlying BCFile.
private BCFile.Writer writerBCF;
-
+
// current data block appender.
BlockAppender blkAppender;
long blkRecordCount;
-
+
// buffers for caching the key.
BoundedByteArrayOutputStream currentKeyBufferOS;
BoundedByteArrayOutputStream lastKeyBufferOS;
-
+
// buffer used by chunk codec
private byte[] valueBuffer;
-
+
/**
* Writer states. The state always transits in circles: READY -> IN_KEY -> END_KEY -> IN_VALUE -> READY.
*/
@@ -205,12 +205,12 @@ public class TFile {
// ERROR, // Error encountered, cannot continue.
CLOSED, // TFile already closed.
}
-
+
// current state of Writer.
State state = State.READY;
Configuration conf;
long errorCount = 0;
-
+
/**
* Constructor
*
@@ -239,18 +239,19 @@ public class TFile {
sizeMinBlock = minBlockSize;
tfileMeta = new TFileMeta(comparator);
tfileIndex = new TFileIndex(tfileMeta.getComparator());
-
+
writerBCF = new BCFile.Writer(fsdos, compressName, conf, true);
currentKeyBufferOS = new BoundedByteArrayOutputStream(MAX_KEY_SIZE);
lastKeyBufferOS = new BoundedByteArrayOutputStream(MAX_KEY_SIZE);
this.conf = conf;
}
-
+
/**
* Close the Writer. Resources will be released regardless of the exceptions being thrown. Future close calls will have no effect.
*
* The underlying FSDataOutputStream is not closed.
*/
+ @Override
public void close() throws IOException {
if ((state == State.CLOSED)) {
return;
@@ -262,9 +263,9 @@ public class TFile {
if (state != State.READY) {
throw new IllegalStateException("Cannot close TFile in the middle of key-value insertion.");
}
-
+
finishDataBlock(true);
-
+
// first, write out data:TFile.meta
BlockAppender outMeta = writerBCF.prepareMetaBlock(TFileMeta.BLOCK_NAME, COMPRESSION_NONE);
try {
@@ -272,7 +273,7 @@ public class TFile {
} finally {
outMeta.close();
}
-
+
// second, write out data:TFile.index
BlockAppender outIndex = writerBCF.prepareMetaBlock(TFileIndex.BLOCK_NAME);
try {
@@ -280,7 +281,7 @@ public class TFile {
} finally {
outIndex.close();
}
-
+
writerBCF.close();
}
} finally {
@@ -290,7 +291,7 @@ public class TFile {
state = State.CLOSED;
}
}
-
+
/**
* Adding a new key-value pair to the TFile. This is synonymous to append(key, 0, key.length, value, 0, value.length)
*
@@ -303,7 +304,7 @@ public class TFile {
public void append(byte[] key, byte[] value) throws IOException {
append(key, 0, key.length, value, 0, value.length);
}
-
+
/**
* Adding a new key-value pair to TFile.
*
@@ -328,11 +329,11 @@ public class TFile {
if ((koff | klen | (koff + klen) | (key.length - (koff + klen))) < 0) {
throw new IndexOutOfBoundsException("Bad key buffer offset-length combination.");
}
-
+
if ((voff | vlen | (voff + vlen) | (value.length - (voff + vlen))) < 0) {
throw new IndexOutOfBoundsException("Bad value buffer offset-length combination.");
}
-
+
try {
DataOutputStream dosKey = prepareAppendKey(klen);
try {
@@ -342,7 +343,7 @@ public class TFile {
} finally {
dosKey.close();
}
-
+
DataOutputStream dosValue = prepareAppendValue(vlen);
try {
++errorCount;
@@ -355,14 +356,14 @@ public class TFile {
state = State.READY;
}
}
-
+
/**
* Helper class to register key after close call on key append stream.
*/
private class KeyRegister extends DataOutputStream {
private final int expectedLength;
private boolean closed = false;
-
+
public KeyRegister(int len) {
super(currentKeyBufferOS);
if (len >= 0) {
@@ -372,13 +373,13 @@ public class TFile {
}
expectedLength = len;
}
-
+
@Override
public void close() throws IOException {
if (closed == true) {
return;
}
-
+
try {
++errorCount;
byte[] key = currentKeyBufferOS.getBuffer();
@@ -389,13 +390,13 @@ public class TFile {
if (expectedLength >= 0 && expectedLength != len) {
throw new IOException("Incorrect key length: expected=" + expectedLength + " actual=" + len);
}
-
+
Utils.writeVInt(blkAppender, len);
blkAppender.write(key, 0, len);
if (tfileIndex.getFirstKey() == null) {
tfileIndex.setFirstKey(key, 0, len);
}
-
+
if (tfileMeta.isSorted()) {
byte[] lastKey = lastKeyBufferOS.getBuffer();
int lastLen = lastKeyBufferOS.size();
@@ -403,7 +404,7 @@ public class TFile {
throw new IOException("Keys are not added in sorted order");
}
}
-
+
BoundedByteArrayOutputStream tmp = currentKeyBufferOS;
currentKeyBufferOS = lastKeyBufferOS;
lastKeyBufferOS = tmp;
@@ -414,29 +415,29 @@ public class TFile {
}
}
}
-
+
/**
* Helper class to register value after close call on value append stream.
*/
private class ValueRegister extends DataOutputStream {
private boolean closed = false;
-
+
public ValueRegister(OutputStream os) {
super(os);
}
-
+
// Avoiding flushing call to down stream.
@Override
public void flush() {
// do nothing
}
-
+
@Override
public void close() throws IOException {
if (closed == true) {
return;
}
-
+
try {
++errorCount;
super.close();
@@ -451,7 +452,7 @@ public class TFile {
}
}
}
-
+
/**
* Obtain an output stream for writing a key into TFile. This may only be called when there is no active Key appending stream or value appending stream.
*
@@ -466,13 +467,13 @@ public class TFile {
if (state != State.READY) {
throw new IllegalStateException("Incorrect state to start a new key: " + state.name());
}
-
+
initDataBlock();
DataOutputStream ret = new KeyRegister(length);
state = State.IN_KEY;
return ret;
}
-
+
/**
* Obtain an output stream for writing a value into TFile. This may only be called right after a key appending operation (the key append stream must be
* closed).
@@ -488,9 +489,9 @@ public class TFile {
if (state != State.END_KEY) {
throw new IllegalStateException("Incorrect state to start a new value: " + state.name());
}
-
+
DataOutputStream ret;
-
+
// unknown length
if (length < 0) {
if (valueBuffer == null) {
@@ -500,11 +501,11 @@ public class TFile {
} else {
ret = new ValueRegister(new Chunk.SingleChunkEncoder(blkAppender, length));
}
-
+
state = State.IN_VALUE;
return ret;
}
-
+
/**
* Obtain an output stream for creating a meta block. This function may not be called when there is a key append stream or value append stream active. No
* more key-value insertion is allowed after a meta data block has been added to TFile.
@@ -522,12 +523,12 @@ public class TFile {
if (state != State.READY) {
throw new IllegalStateException("Incorrect state to start a Meta Block: " + state.name());
}
-
+
finishDataBlock(true);
DataOutputStream outputStream = writerBCF.prepareMetaBlock(name, compressName);
return outputStream;
}
-
+
/**
* Obtain an output stream for creating a meta block. This function may not be called when there is a key append stream or value append stream active. No
* more key-value insertion is allowed after a meta data block has been added to TFile. Data will be compressed using the default compressor as defined in
@@ -544,11 +545,11 @@ public class TFile {
if (state != State.READY) {
throw new IllegalStateException("Incorrect state to start a Meta Block: " + state.name());
}
-
+
finishDataBlock(true);
return writerBCF.prepareMetaBlock(name);
}
-
+
/**
* Check if we need to start a new data block.
*
@@ -560,7 +561,7 @@ public class TFile {
blkAppender = writerBCF.prepareDataBlock();
}
}
-
+
/**
* Close the current data block if necessary.
*
@@ -572,7 +573,7 @@ public class TFile {
if (blkAppender == null) {
return;
}
-
+
// exceeded the size limit, do the compression and finish the block
if (bForceFinish || blkAppender.getCompressedSize() >= sizeMinBlock) {
// keep tracks of the last key of each data block, no padding
@@ -586,7 +587,7 @@ public class TFile {
}
}
}
-
+
/**
* TFile Reader. Users may only read TFiles by creating TFile.Reader.Scanner. objects. A scanner may scan the whole TFile ({@link Reader#createScanner()} ) ,
* a portion of TFile based on byte offsets ( {@link Reader#createScanner(long, long)}), or a portion of TFile with keys fall in a certain key range (for
@@ -595,16 +596,16 @@ public class TFile {
public static class Reader implements Closeable {
// The underlying BCFile reader.
final BCFile.Reader readerBCF;
-
+
// TFile index, it is loaded lazily.
TFileIndex tfileIndex = null;
final TFileMeta tfileMeta;
final BytesComparator comparator;
-
+
// global begin and end locations.
private final Location begin;
private final Location end;
-
+
/**
* Location representing a virtual position in the TFile.
*/
@@ -612,27 +613,27 @@ public class TFile {
private int blockIndex;
// distance/offset from the beginning of the block
private long recordIndex;
-
+
Location(int blockIndex, long recordIndex) {
set(blockIndex, recordIndex);
}
-
+
void incRecordIndex() {
++recordIndex;
}
-
+
Location(Location other) {
set(other);
}
-
+
int getBlockIndex() {
return blockIndex;
}
-
+
long getRecordIndex() {
return recordIndex;
}
-
+
void set(int blockIndex, long recordIndex) {
if ((blockIndex | recordIndex) < 0) {
throw new IllegalArgumentException("Illegal parameter for BlockLocation.");
@@ -640,11 +641,11 @@ public class TFile {
this.blockIndex = blockIndex;
this.recordIndex = recordIndex;
}
-
+
void set(Location other) {
set(other.blockIndex, other.recordIndex);
}
-
+
/**
* @see java.lang.Comparable#compareTo(java.lang.Object)
*/
@@ -652,7 +653,7 @@ public class TFile {
public int compareTo(Location other) {
return compareTo(other.blockIndex, other.recordIndex);
}
-
+
int compareTo(int bid, long rid) {
if (this.blockIndex == bid) {
long ret = this.recordIndex - rid;
@@ -664,7 +665,7 @@ public class TFile {
}
return this.blockIndex - bid;
}
-
+
/**
* @see java.lang.Object#clone()
*/
@@ -672,7 +673,7 @@ public class TFile {
protected Location clone() {
return new Location(blockIndex, recordIndex);
}
-
+
/**
* @see java.lang.Object#hashCode()
*/
@@ -683,7 +684,7 @@ public class TFile {
result = (int) (prime * result + recordIndex);
return result;
}
-
+
/**
* @see java.lang.Object#equals(java.lang.Object)
*/
@@ -703,7 +704,7 @@ public class TFile {
return true;
}
}
-
+
/**
* Constructor
*
@@ -716,7 +717,7 @@ public class TFile {
*/
public Reader(FSDataInputStream fsdis, long fileLength, Configuration conf) throws IOException {
readerBCF = new BCFile.Reader(fsdis, fileLength, conf);
-
+
// first, read TFile meta
BlockReader brMeta = readerBCF.getMetaBlock(TFileMeta.BLOCK_NAME);
try {
@@ -724,20 +725,21 @@ public class TFile {
} finally {
brMeta.close();
}
-
+
comparator = tfileMeta.getComparator();
// Set begin and end locations.
begin = new Location(0, 0);
end = new Location(readerBCF.getBlockCount(), 0);
}
-
+
/**
* Close the reader. The state of the Reader object is undefined after close. Calling close() for multiple times has no effect.
*/
+ @Override
public void close() throws IOException {
readerBCF.close();
}
-
+
/**
* Get the begin location of the TFile.
*
@@ -746,7 +748,7 @@ public class TFile {
Location begin() {
return begin;
}
-
+
/**
* Get the end location of the TFile.
*
@@ -755,7 +757,7 @@ public class TFile {
Location end() {
return end;
}
-
+
/**
* Get the string representation of the comparator.
*
@@ -765,7 +767,7 @@ public class TFile {
public String getComparatorName() {
return tfileMeta.getComparatorString();
}
-
+
/**
* Is the TFile sorted?
*
@@ -774,7 +776,7 @@ public class TFile {
public boolean isSorted() {
return tfileMeta.isSorted();
}
-
+
/**
* Get the number of key-value pair entries in TFile.
*
@@ -783,7 +785,7 @@ public class TFile {
public long getEntryCount() {
return tfileMeta.getRecordCount();
}
-
+
/**
* Lazily loading the TFile index.
*
@@ -799,7 +801,7 @@ public class TFile {
}
}
}
-
+
/**
* Get the first key in the TFile.
*
@@ -810,7 +812,7 @@ public class TFile {
checkTFileDataIndex();
return tfileIndex.getFirstKey();
}
-
+
/**
* Get the last key in the TFile.
*
@@ -821,7 +823,7 @@ public class TFile {
checkTFileDataIndex();
return tfileIndex.getLastKey();
}
-
+
/**
* Get a Comparator object to compare Entries. It is useful when you want stores the entries in a collection (such as PriorityQueue) and perform sorting or
* comparison among entries based on the keys without copying out the key.
@@ -832,7 +834,7 @@ public class TFile {
if (!isSorted()) {
throw new RuntimeException("Entries are not comparable for unsorted TFiles");
}
-
+
return new Comparator<Scanner.Entry>() {
/**
* Provide a customized comparator for Entries. This is useful if we have a collection of Entry objects. However, if the Entry objects come from
@@ -844,7 +846,7 @@ public class TFile {
}
};
}
-
+
/**
* Get an instance of the RawComparator that is constructed based on the string comparator representation.
*
@@ -853,7 +855,7 @@ public class TFile {
public Comparator<RawComparable> getComparator() {
return comparator;
}
-
+
/**
* Stream access to a meta block.``
*
@@ -868,7 +870,7 @@ public class TFile {
public DataInputStream getMetaBlock(String name) throws IOException, MetaBlockDoesNotExist {
return readerBCF.getMetaBlock(name);
}
-
+
/**
* if greater is true then returns the beginning location of the block containing the key strictly greater than input key. if greater is false then returns
* the beginning location of the block greater than equal to the input key
@@ -877,7 +879,6 @@ public class TFile {
* the input key
* @param greater
* boolean flag
- * @return
* @throws IOException
*/
Location getBlockContainsKey(RawComparable key, boolean greater) throws IOException {
@@ -890,21 +891,21 @@ public class TFile {
return end;
return new Location(blkIndex, 0);
}
-
+
int compareKeys(byte[] a, int o1, int l1, byte[] b, int o2, int l2) {
if (!isSorted()) {
throw new RuntimeException("Cannot compare keys for unsorted TFiles.");
}
return comparator.compare(a, o1, l1, b, o2, l2);
}
-
+
int compareKeys(RawComparable a, RawComparable b) {
if (!isSorted()) {
throw new RuntimeException("Cannot compare keys for unsorted TFiles.");
}
return comparator.compare(a, b);
}
-
+
/**
* Get the location pointing to the beginning of the first key-value pair in a compressed block whose byte offset in the TFile is greater than or equal to
* the specified offset.
@@ -919,7 +920,7 @@ public class TFile {
return end;
return new Location(blockIndex, 0);
}
-
+
/**
* Get a sample key that is within a block whose starting offset is greater than or equal to the specified offset.
*
@@ -935,7 +936,7 @@ public class TFile {
checkTFileDataIndex();
return new ByteArray(tfileIndex.getEntry(blockIndex).key);
}
-
+
/**
* Get a scanner than can scan the whole TFile.
*
@@ -945,7 +946,7 @@ public class TFile {
public Scanner createScanner() throws IOException {
return new Scanner(this, begin, end);
}
-
+
/**
* Get a scanner that covers a portion of TFile based on byte offsets.
*
@@ -960,7 +961,7 @@ public class TFile {
public Scanner createScanner(long offset, long length) throws IOException {
return new Scanner(this, offset, offset + length);
}
-
+
/**
* Get a scanner that covers a portion of TFile based on keys.
*
@@ -975,7 +976,7 @@ public class TFile {
return createScanner((beginKey == null) ? null : new ByteArray(beginKey, 0, beginKey.length), (endKey == null) ? null : new ByteArray(endKey, 0,
endKey.length));
}
-
+
/**
* Get a scanner that covers a specific key range.
*
@@ -992,7 +993,7 @@ public class TFile {
}
return new Scanner(this, beginKey, endKey);
}
-
+
/**
* The TFile Scanner. The Scanner has an implicit cursor, which, upon creation, points to the first key-value pair in the scan range. If the scan range is
* empty, the cursor will point to the end of the scan range.
@@ -1010,27 +1011,27 @@ public class TFile {
final Reader reader;
// current block (null if reaching end)
private BlockReader blkReader;
-
+
Location beginLocation;
Location endLocation;
Location currentLocation;
-
+
// flag to ensure value is only examined once.
boolean valueChecked = false;
// reusable buffer for keys.
final byte[] keyBuffer;
// length of key, -1 means key is invalid.
int klen = -1;
-
+
static final int MAX_VAL_TRANSFER_BUF_SIZE = 128 * 1024;
BytesWritable valTransferBuffer;
-
+
DataInputBuffer keyDataInputStream;
ChunkDecoder valueBufferInputStream;
DataInputStream valueDataInputStream;
// vlen == -1 if unknown.
int vlen;
-
+
/**
* Constructor
*
@@ -1047,7 +1048,7 @@ public class TFile {
protected Scanner(Reader reader, long offBegin, long offEnd) throws IOException {
this(reader, reader.getLocationNear(offBegin), reader.getLocationNear(offEnd));
}
-
+
/**
* Constructor
*
@@ -1065,13 +1066,13 @@ public class TFile {
reader.checkTFileDataIndex();
beginLocation = begin;
endLocation = end;
-
+
valTransferBuffer = new BytesWritable();
keyBuffer = new byte[MAX_KEY_SIZE];
keyDataInputStream = new DataInputBuffer();
valueBufferInputStream = new ChunkDecoder();
valueDataInputStream = new DataInputStream(valueBufferInputStream);
-
+
if (beginLocation.compareTo(endLocation) >= 0) {
currentLocation = new Location(endLocation);
} else {
@@ -1080,7 +1081,7 @@ public class TFile {
inBlockAdvance(beginLocation.getRecordIndex());
}
}
-
+
/**
* Constructor
*
@@ -1104,7 +1105,7 @@ public class TFile {
seekTo(beginLocation);
}
}
-
+
/**
* Move the cursor to the first entry whose key is greater than or equal to the input key. Synonymous to seekTo(key, 0, key.length). The entry returned by
* the previous entry() call will be invalid.
@@ -1117,7 +1118,7 @@ public class TFile {
public boolean seekTo(byte[] key) throws IOException {
return seekTo(key, 0, key.length);
}
-
+
/**
* Move the cursor to the first entry whose key is greater than or equal to the input key. The entry returned by the previous entry() call will be
* invalid.
@@ -1134,7 +1135,7 @@ public class TFile {
public boolean seekTo(byte[] key, int keyOffset, int keyLen) throws IOException {
return seekTo(new ByteArray(key, keyOffset, keyLen), false);
}
-
+
private boolean seekTo(RawComparable key, boolean beyond) throws IOException {
Location l = reader.getBlockContainsKey(key, beyond);
if (l.compareTo(beginLocation) < 0) {
@@ -1143,17 +1144,17 @@ public class TFile {
seekTo(endLocation);
return false;
}
-
+
// check if what we are seeking is in the later part of the current
// block.
if (atEnd() || (l.getBlockIndex() != currentLocation.getBlockIndex()) || (compareCursorKeyTo(key) >= 0)) {
// sorry, we must seek to a different location first.
seekTo(l);
}
-
+
return inBlockAdvance(key, beyond);
}
-
+
/**
* Move the cursor to the new location. The entry returned by the previous entry() call will be invalid.
*
@@ -1165,16 +1166,16 @@ public class TFile {
if (l.compareTo(beginLocation) < 0) {
throw new IllegalArgumentException("Attempt to seek before the begin location.");
}
-
+
if (l.compareTo(endLocation) > 0) {
throw new IllegalArgumentException("Attempt to seek after the end location.");
}
-
+
if (l.compareTo(endLocation) == 0) {
parkCursorAtEnd();
return;
}
-
+
if (l.getBlockIndex() != currentLocation.getBlockIndex()) {
// going to a totally different block
initBlock(l.getBlockIndex());
@@ -1188,12 +1189,12 @@ public class TFile {
initBlock(l.getBlockIndex());
}
}
-
+
inBlockAdvance(l.getRecordIndex() - currentLocation.getRecordIndex());
-
+
return;
}
-
+
/**
* Rewind to the first entry in the scanner. The entry returned by the previous entry() call will be invalid.
*
@@ -1202,7 +1203,7 @@ public class TFile {
public void rewind() throws IOException {
seekTo(beginLocation);
}
-
+
/**
* Seek to the end of the scanner. The entry returned by the previous entry() call will be invalid.
*
@@ -1211,7 +1212,7 @@ public class TFile {
public void seekToEnd() throws IOException {
parkCursorAtEnd();
}
-
+
/**
* Move the cursor to the first entry whose key is greater than or equal to the input key. Synonymous to lowerBound(key, 0, key.length). The entry
* returned by the previous entry() call will be invalid.
@@ -1223,7 +1224,7 @@ public class TFile {
public void lowerBound(byte[] key) throws IOException {
lowerBound(key, 0, key.length);
}
-
+
/**
* Move the cursor to the first entry whose key is greater than or equal to the input key. The entry returned by the previous entry() call will be
* invalid.
@@ -1239,7 +1240,7 @@ public class TFile {
public void lowerBound(byte[] key, int keyOffset, int keyLen) throws IOException {
seekTo(new ByteArray(key, keyOffset, keyLen), false);
}
-
+
/**
* Move the cursor to the first entry whose key is strictly greater than the input key. Synonymous to upperBound(key, 0, key.length). The entry returned
* by the previous entry() call will be invalid.
@@ -1251,7 +1252,7 @@ public class TFile {
public void upperBound(byte[] key) throws IOException {
upperBound(key, 0, key.length);
}
-
+
/**
* Move the cursor to the first entry whose key is strictly greater than the input key. The entry returned by the previous entry() call will be invalid.
*
@@ -1266,7 +1267,7 @@ public class TFile {
public void upperBound(byte[] key, int keyOffset, int keyLen) throws IOException {
seekTo(new ByteArray(key, keyOffset, keyLen), true);
}
-
+
/**
* Move the cursor to the next key-value pair. The entry returned by the previous entry() call will be invalid.
*
@@ -1277,7 +1278,7 @@ public class TFile {
if (atEnd()) {
return false;
}
-
+
int curBid = currentLocation.getBlockIndex();
long curRid = currentLocation.getRecordIndex();
long entriesInBlock = reader.getBlockEntryCount(curBid);
@@ -1294,7 +1295,7 @@ public class TFile {
}
return true;
}
-
+
/**
* Load a compressed block for reading. Expecting blockIndex is valid.
*
@@ -1312,7 +1313,7 @@ public class TFile {
blkReader = reader.getBlockReader(blockIndex);
currentLocation.set(blockIndex, 0);
}
-
+
private void parkCursorAtEnd() throws IOException {
klen = -1;
currentLocation.set(endLocation);
@@ -1324,15 +1325,16 @@ public class TFile {
}
}
}
-
+
/**
* Close the scanner. Release all resources. The behavior of using the scanner after calling close is not defined. The entry returned by the previous
* entry() call will be invalid.
*/
+ @Override
public void close() throws IOException {
parkCursorAtEnd();
}
-
+
/**
* Is cursor at the end location?
*
@@ -1341,7 +1343,7 @@ public class TFile {
public boolean atEnd() {
return (currentLocation.compareTo(endLocation) >= 0);
}
-
+
/**
* check whether we have already successfully obtained the key. It also initializes the valueInputStream.
*/
@@ -1354,7 +1356,7 @@ public class TFile {
klen = -1;
vlen = -1;
valueChecked = false;
-
+
klen = Utils.readVInt(blkReader);
blkReader.readFully(keyBuffer, 0, klen);
valueBufferInputStream.reset(blkReader);
@@ -1362,7 +1364,7 @@ public class TFile {
vlen = valueBufferInputStream.getRemain();
}
}
-
+
/**
* Get an entry to access the key and value.
*
@@ -1373,7 +1375,7 @@ public class TFile {
checkKey();
return new Entry();
}
-
+
/**
* Internal API. Comparing the key at cursor to user-specified key.
*
@@ -1386,7 +1388,7 @@ public class TFile {
checkKey();
return reader.compareKeys(keyBuffer, 0, klen, other.buffer(), other.offset(), other.size());
}
-
+
/**
* Entry to a <Key, Value> pair.
*/
@@ -1399,11 +1401,11 @@ public class TFile {
public int getKeyLength() {
return klen;
}
-
+
byte[] getKeyBuffer() {
return keyBuffer;
}
-
+
/**
* Copy the key and value in one shot into BytesWritables. This is equivalent to getKey(key); getValue(value);
*
@@ -1417,7 +1419,7 @@ public class TFile {
getKey(key);
getValue(value);
}
-
+
/**
* Copy the key into BytesWritable. The input BytesWritable will be automatically resized to the actual key size.
*
@@ -1430,7 +1432,7 @@ public class TFile {
getKey(key.getBytes());
return key.getLength();
}
-
+
/**
* Copy the value into BytesWritable. The input BytesWritable will be automatically resized to the actual value size. The implementation directly uses
* the buffer inside BytesWritable for storing the value. The call does not require the value length to be known.
@@ -1453,7 +1455,7 @@ public class TFile {
dis.close();
}
}
-
+
/**
* Writing the key to the output stream. This method avoids copying key buffer from Scanner into user buffer, then writing to the output stream.
*
@@ -1466,7 +1468,7 @@ public class TFile {
out.write(keyBuffer, 0, klen);
return klen;
}
-
+
/**
* Writing the value to the output stream. This method avoids copying value data from Scanner into user buffer, then writing to the output stream. It
* does not require the value length to be known.
@@ -1493,7 +1495,7 @@ public class TFile {
dis.close();
}
}
-
+
/**
* Copy the key into user supplied buffer.
*
@@ -1506,7 +1508,7 @@ public class TFile {
public int getKey(byte[] buf) throws IOException {
return getKey(buf, 0);
}
-
+
/**
* Copy the key into user supplied buffer.
*
@@ -1525,7 +1527,7 @@ public class TFile {
System.arraycopy(keyBuffer, 0, buf, offset, klen);
return klen;
}
-
+
/**
* Streaming access to the key. Useful for desrializing the key into user objects.
*
@@ -1535,7 +1537,7 @@ public class TFile {
keyDataInputStream.reset(keyBuffer, klen);
return keyDataInputStream;
}
-
+
/**
* Get the length of the value. isValueLengthKnown() must be tested true.
*
@@ -1545,10 +1547,10 @@ public class TFile {
if (vlen >= 0) {
return vlen;
}
-
+
throw new RuntimeException("Value length unknown.");
}
-
+
/**
* Copy value into user-supplied buffer. User supplied buffer must be large enough to hold the whole value. The value part of the key-value pair pointed
* by the current cursor is not cached and can only be examined once. Calling any of the following functions more than once without moving the cursor
@@ -1561,7 +1563,7 @@ public class TFile {
public int getValue(byte[] buf) throws IOException {
return getValue(buf, 0);
}
-
+
/**
* Copy value into user-supplied buffer. User supplied buffer must be large enough to hold the whole value (starting from the offset). The value part of
* the key-value pair pointed by the current cursor is not cached and can only be examined once. Calling any of the following functions more than once
@@ -1580,7 +1582,7 @@ public class TFile {
dis.readFully(buf, offset, vlen);
return vlen;
}
-
+
int nextOffset = offset;
while (nextOffset < buf.length) {
int n = dis.read(buf, nextOffset, buf.length - nextOffset);
@@ -1600,7 +1602,7 @@ public class TFile {
dis.close();
}
}
-
+
/**
* Stream access to value. The value part of the key-value pair pointed by the current cursor is not cached and can only be examined once. Calling any
* of the following functions more than once without moving the cursor will result in exception: {@link #getValue(byte[])},
@@ -1616,7 +1618,7 @@ public class TFile {
valueChecked = true;
return valueDataInputStream;
}
-
+
/**
* Check whether it is safe to call getValueLength().
*
@@ -1626,7 +1628,7 @@ public class TFile {
public boolean isValueLengthKnown() {
return (vlen >= 0);
}
-
+
/**
* Compare the entry key to another key. Synonymous to compareTo(key, 0, key.length).
*
@@ -1637,7 +1639,7 @@ public class TFile {
public int compareTo(byte[] buf) {
return compareTo(buf, 0, buf.length);
}
-
+
/**
* Compare the entry key to another key. Synonymous to compareTo(new ByteArray(buf, offset, length)
*
@@ -1652,7 +1654,7 @@ public class TFile {
public int compareTo(byte[] buf, int offset, int length) {
return compareTo(new ByteArray(buf, offset, length));
}
-
+
/**
* Compare an entry with a RawComparable object. This is useful when Entries are stored in a collection, and we want to compare a user supplied key.
*/
@@ -1660,7 +1662,7 @@ public class TFile {
public int compareTo(RawComparable key) {
return reader.compareKeys(keyBuffer, 0, getKeyLength(), key.buffer(), key.offset(), key.size());
}
-
+
/**
* Compare whether this and other points to the same key value.
*/
@@ -1672,13 +1674,13 @@ public class TFile {
return false;
return ((Entry) other).compareTo(keyBuffer, 0, getKeyLength()) == 0;
}
-
+
@Override
public int hashCode() {
return WritableComparator.hashBytes(keyBuffer, getKeyLength());
}
}
-
+
/**
* Advance cursor by n positions within the block.
*
@@ -1696,7 +1698,7 @@ public class TFile {
currentLocation.incRecordIndex();
}
}
-
+
/**
* Advance cursor in block until we find a key that is greater than or equal to the input key.
*
@@ -1713,7 +1715,7 @@ public class TFile {
if (curBid == endLocation.getBlockIndex()) {
entryInBlock = endLocation.getRecordIndex();
}
-
+
while (currentLocation.getRecordIndex() < entryInBlock) {
int cmp = compareCursorKeyTo(key);
if (cmp > 0)
@@ -1726,20 +1728,20 @@ public class TFile {
klen = -1;
currentLocation.incRecordIndex();
}
-
+
throw new RuntimeException("Cannot find matching key in block.");
}
}
-
+
long getBlockEntryCount(int curBid) {
return tfileIndex.getEntry(curBid).entries();
}
-
+
BlockReader getBlockReader(int blockIndex) throws IOException {
return readerBCF.getDataBlock(blockIndex);
}
}
-
+
/**
* Data structure representing "TFile.meta" meta block.
*/
@@ -1749,7 +1751,7 @@ public class TFile {
private long recordCount;
private final String strComparator;
private final BytesComparator comparator;
-
+
// ctor for writes
public TFileMeta(String comparator) {
// set fileVersion to API version when we create it.
@@ -1758,7 +1760,7 @@ public class TFile {
strComparator = (comparator == null) ? "" : comparator;
this.comparator = makeComparator(strComparator);
}
-
+
// ctor for reads
public TFileMeta(DataInput in) throws IOException {
version = new Version(in);
@@ -1769,7 +1771,7 @@ public class TFile {
strComparator = Utils.readString(in);
comparator = makeComparator(strComparator);
}
-
+
@SuppressWarnings({"rawtypes", "unchecked"})
static BytesComparator makeComparator(String comparator) {
if (comparator.length() == 0) {
@@ -1792,38 +1794,38 @@ public class TFile {
throw new IllegalArgumentException("Unsupported comparator: " + comparator);
}
}
-
+
public void write(DataOutput out) throws IOException {
TFile.API_VERSION.write(out);
Utils.writeVLong(out, recordCount);
Utils.writeString(out, strComparator);
}
-
+
public long getRecordCount() {
return recordCount;
}
-
+
public void incRecordCount() {
++recordCount;
}
-
+
public boolean isSorted() {
return !strComparator.equals("");
}
-
+
public String getComparatorString() {
return strComparator;
}
-
+
public BytesComparator getComparator() {
return comparator;
}
-
+
public Version getVersion() {
return version;
}
} // END: class MetaTFileMeta
-
+
/**
* Data structure representing "TFile.index" meta block.
*/
@@ -1832,7 +1834,7 @@ public class TFile {
private ByteArray firstKey;
private final ArrayList<TFileIndexEntry> index;
private final BytesComparator comparator;
-
+
/**
* For reading from file.
*
@@ -1845,11 +1847,11 @@ public class TFile {
byte[] buffer = new byte[size];
in.readFully(buffer);
DataInputStream firstKeyInputStream = new DataInputStream(new ByteArrayInputStream(buffer, 0, size));
-
+
int firstKeyLength = Utils.readVInt(firstKeyInputStream);
firstKey = new ByteArray(new byte[firstKeyLength]);
firstKeyInputStream.readFully(firstKey.buffer());
-
+
for (int i = 0; i < entryCount; i++) {
size = Utils.readVInt(in);
if (buffer.length < size) {
@@ -1866,7 +1868,7 @@ public class TFile {
}
this.comparator = comparator;
}
-
+
/**
* @param key
* input key.
@@ -1876,34 +1878,34 @@ public class TFile {
if (comparator == null) {
throw new RuntimeException("Cannot search in unsorted TFile");
}
-
+
if (firstKey == null) {
return -1; // not found
}
-
+
int ret = Utils.lowerBound(index, key, comparator);
if (ret == index.size()) {
return -1;
}
return ret;
}
-
+
public int upperBound(RawComparable key) {
if (comparator == null) {
throw new RuntimeException("Cannot search in unsorted TFile");
}
-
+
if (firstKey == null) {
return -1; // not found
}
-
+
int ret = Utils.upperBound(index, key, comparator);
if (ret == index.size()) {
return -1;
}
return ret;
}
-
+
/**
* For writing to file.
*/
@@ -1911,43 +1913,43 @@ public class TFile {
index = new ArrayList<TFileIndexEntry>();
this.comparator = comparator;
}
-
+
public RawComparable getFirstKey() {
return firstKey;
}
-
+
public void setFirstKey(byte[] key, int offset, int length) {
firstKey = new ByteArray(new byte[length]);
System.arraycopy(key, offset, firstKey.buffer(), 0, length);
}
-
+
public RawComparable getLastKey() {
if (index.size() == 0) {
return null;
}
return new ByteArray(index.get(index.size() - 1).buffer());
}
-
+
public void addEntry(TFileIndexEntry keyEntry) {
index.add(keyEntry);
}
-
+
public TFileIndexEntry getEntry(int bid) {
return index.get(bid);
}
-
+
public void write(DataOutput out) throws IOException {
if (firstKey == null) {
Utils.writeVInt(out, 0);
return;
}
-
+
DataOutputBuffer dob = new DataOutputBuffer();
Utils.writeVInt(dob, firstKey.size());
dob.write(firstKey.buffer());
Utils.writeVInt(out, dob.size());
out.write(dob.getData(), 0, dob.getLength());
-
+
for (TFileIndexEntry entry : index) {
dob.reset();
entry.write(dob);
@@ -1956,7 +1958,7 @@ public class TFile {
}
}
}
-
+
/**
* TFile Data Index entry. We should try to make the memory footprint of each index entry as small as possible.
*/
@@ -1964,47 +1966,47 @@ public class TFile {
final byte[] key;
// count of <key, value> entries in the block.
final long kvEntries;
-
+
public TFileIndexEntry(DataInput in) throws IOException {
int len = Utils.readVInt(in);
key = new byte[len];
in.readFully(key, 0, len);
kvEntries = Utils.readVLong(in);
}
-
+
// default entry, without any padding
public TFileIndexEntry(byte[] newkey, int offset, int len, long entries) {
key = new byte[len];
System.arraycopy(newkey, offset, key, 0, len);
this.kvEntries = entries;
}
-
+
@Override
public byte[] buffer() {
return key;
}
-
+
@Override
public int offset() {
return 0;
}
-
+
@Override
public int size() {
return key.length;
}
-
+
long entries() {
return kvEntries;
}
-
+
public void write(DataOutput out) throws IOException {
Utils.writeVInt(out, key.length);
out.write(key, 0, key.length);
Utils.writeVLong(out, kvEntries);
}
}
-
+
/**
* Dumping the TFile information.
*
@@ -2018,7 +2020,7 @@ public class TFile {
System.exit(0);
}
Configuration conf = new Configuration();
-
+
for (String file : args) {
System.out.println("===" + file + "===");
try {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java
index 2fa3121..ff8a55b 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java
@@ -38,6 +38,7 @@ import org.apache.log4j.Logger;
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.Combiner}
*/
+@Deprecated
public class AggregatingIterator implements SortedKeyValueIterator<Key,Value>, OptionDescriber {
private SortedKeyValueIterator<Key,Value> iterator;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/FamilyIntersectingIterator.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/FamilyIntersectingIterator.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/FamilyIntersectingIterator.java
index 8efd9fd..fee1ca8 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/FamilyIntersectingIterator.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/FamilyIntersectingIterator.java
@@ -23,6 +23,7 @@ import org.apache.accumulo.core.iterators.user.IndexedDocIterator;
*
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.user.IndexedDocIterator}
*/
+@Deprecated
public class FamilyIntersectingIterator extends IndexedDocIterator {
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/FilteringIterator.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/FilteringIterator.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/FilteringIterator.java
index 723ad55..a200cf2 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/FilteringIterator.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/FilteringIterator.java
@@ -36,6 +36,7 @@ import org.apache.log4j.Logger;
/**
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.Filter}
**/
+@Deprecated
public class FilteringIterator extends WrappingIterator implements OptionDescriber {
private List<? extends Filter> filters;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/FirstEntryInRowIterator.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/FirstEntryInRowIterator.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/FirstEntryInRowIterator.java
index 3ffa0d1..3b96e2e 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/FirstEntryInRowIterator.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/FirstEntryInRowIterator.java
@@ -50,6 +50,7 @@ public class FirstEntryInRowIterator extends SkippingIterator implements OptionD
* @param iteratorName
* @param num
*/
+ @Deprecated
public static void setNumScansBeforeSeek(Scanner scanner, String iteratorName, int num) {
scanner.setScanIteratorOption(iteratorName, NUM_SCANS_STRING_NAME, Integer.toString(num));
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/GrepIterator.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/GrepIterator.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/GrepIterator.java
index 7e4c015..f7a68e9 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/GrepIterator.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/GrepIterator.java
@@ -21,6 +21,7 @@ package org.apache.accumulo.core.iterators;
*
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.user.GrepIterator}
*/
+@Deprecated
public class GrepIterator extends org.apache.accumulo.core.iterators.user.GrepIterator {
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/IntersectingIterator.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/IntersectingIterator.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/IntersectingIterator.java
index b52a0b9..3f6ee9f 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/IntersectingIterator.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/IntersectingIterator.java
@@ -21,6 +21,7 @@ package org.apache.accumulo.core.iterators;
*
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.user.IntersectingIterator}
*/
+@Deprecated
public class IntersectingIterator extends org.apache.accumulo.core.iterators.user.IntersectingIterator {
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
index 72f850d..c7fd5ed 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
@@ -68,6 +68,7 @@ public class IteratorUtil {
*
* @see TableOperations#attachIterator(String, IteratorSetting)
*/
+ @Deprecated
public static Map<String,String> generateAggTableProperties(List<? extends PerColumnIteratorConfig> aggregators) {
Map<String,String> props = new TreeMap<String,String>();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/LargeRowFilter.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/LargeRowFilter.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/LargeRowFilter.java
index be60b50..4194202 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/LargeRowFilter.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/LargeRowFilter.java
@@ -21,6 +21,7 @@ package org.apache.accumulo.core.iterators;
*
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.user.LargeRowFilter}
*/
+@Deprecated
public class LargeRowFilter extends org.apache.accumulo.core.iterators.user.LargeRowFilter {
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/NoLabelIterator.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/NoLabelIterator.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/NoLabelIterator.java
index 3bcefa6..ee47ca0 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/NoLabelIterator.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/NoLabelIterator.java
@@ -26,6 +26,7 @@ import org.apache.accumulo.core.iterators.filter.NoLabelFilter;
/**
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.user.ReqVisFilter}
**/
+@Deprecated
public class NoLabelIterator extends SkippingIterator implements OptionDescriber {
private NoLabelFilter ref = new NoLabelFilter();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/RegExIterator.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/RegExIterator.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/RegExIterator.java
index 2887a89..1ebfa31 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/RegExIterator.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/RegExIterator.java
@@ -27,6 +27,7 @@ import org.apache.accumulo.core.iterators.filter.RegExFilter;
/**
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.user.RegExFilter}
**/
+@Deprecated
public class RegExIterator extends SkippingIterator implements OptionDescriber {
private RegExFilter ref = new RegExFilter();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/RowDeletingIterator.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/RowDeletingIterator.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/RowDeletingIterator.java
index 60ccedd..6785b28 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/RowDeletingIterator.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/RowDeletingIterator.java
@@ -21,6 +21,7 @@ package org.apache.accumulo.core.iterators;
*
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.user.RowDeletingIterator}
*/
+@Deprecated
public class RowDeletingIterator extends org.apache.accumulo.core.iterators.user.RowDeletingIterator {
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/SortedKeyValueIterator.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/SortedKeyValueIterator.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/SortedKeyValueIterator.java
index 3503a43..5b6c4ec 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/SortedKeyValueIterator.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/SortedKeyValueIterator.java
@@ -82,11 +82,11 @@ public interface SortedKeyValueIterator<K extends WritableComparable<?>,V extend
* to a key in the middle of such a group (e.g. the middle of a row). Even if the client always seeks to a range containing an entire group (a,c), the tablet
* server could send back a batch of entries corresponding to (a,b], then reseek the iterator to range (b,c) when the scan is continued.
*
- * {@link columnFamilies} is used, at the lowest level, to determine which data blocks inside of an RFile need to be opened for this iterator. This set of data
+ * {@code columnFamilies} is used, at the lowest level, to determine which data blocks inside of an RFile need to be opened for this iterator. This set of data
* blocks is also the set of locality groups defined for the given table. If no columnFamilies are provided, the data blocks for all locality groups inside of
- * the correct RFile will be opened and seeked in an attempt to find the correct start key, irregardless of the startKey in the {@link range}.
+ * the correct RFile will be opened and seeked in an attempt to find the correct start key, regardless of the startKey in the {@code range}.
*
- * In an Accumulo instance in which multiple locality groups exist for a table, it is important to ensure that {@link columnFamilies} is properly set to the
+ * In an Accumulo instance in which multiple locality groups exist for a table, it is important to ensure that {@code columnFamilies} is properly set to the
* minimum required column families to ensure that data from separate locality groups is not inadvertently read.
*
* @param range
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/VersioningIterator.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/VersioningIterator.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/VersioningIterator.java
index 38f56ca..cc37067 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/VersioningIterator.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/VersioningIterator.java
@@ -24,6 +24,7 @@ import org.apache.accumulo.core.data.Value;
*
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.user.VersioningIterator}
*/
+@Deprecated
public class VersioningIterator extends org.apache.accumulo.core.iterators.user.VersioningIterator {
public VersioningIterator() {}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/WholeRowIterator.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/WholeRowIterator.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/WholeRowIterator.java
index 0ea97a0..326ec63 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/WholeRowIterator.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/WholeRowIterator.java
@@ -21,6 +21,7 @@ package org.apache.accumulo.core.iterators;
*
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.user.WholeRowIterator}
*/
+@Deprecated
public class WholeRowIterator extends org.apache.accumulo.core.iterators.user.WholeRowIterator {
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/Aggregator.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/Aggregator.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/Aggregator.java
index 83f0f39..2264bd9 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/Aggregator.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/Aggregator.java
@@ -21,6 +21,7 @@ import org.apache.accumulo.core.data.Value;
/**
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.Combiner}
*/
+@Deprecated
public interface Aggregator {
void reset();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/LongSummation.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/LongSummation.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/LongSummation.java
index 3da9a01..bee96ac 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/LongSummation.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/LongSummation.java
@@ -25,6 +25,7 @@ import org.apache.log4j.Logger;
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.user.SummingCombiner} with
* {@link org.apache.accumulo.core.iterators.LongCombiner.Type#FIXEDLEN}
*/
+@Deprecated
public class LongSummation implements Aggregator {
private static final Logger log = Logger.getLogger(LongSummation.class);
long sum = 0;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/NumArraySummation.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/NumArraySummation.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/NumArraySummation.java
index 6a93f22..1d1e4cb 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/NumArraySummation.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/NumArraySummation.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.io.WritableUtils;
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.user.SummingArrayCombiner} with
* {@link org.apache.accumulo.core.iterators.user.SummingArrayCombiner.Type#VARLEN}
*/
+@Deprecated
public class NumArraySummation implements Aggregator {
long[] sum = new long[0];
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/NumSummation.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/NumSummation.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/NumSummation.java
index 2fa6026..7994195 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/NumSummation.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/NumSummation.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.io.WritableUtils;
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.user.SummingCombiner} with
* {@link org.apache.accumulo.core.iterators.LongCombiner.Type#VARLEN}
*/
+@Deprecated
public class NumSummation implements Aggregator {
long sum = 0l;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/StringMax.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/StringMax.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/StringMax.java
index f066f7c..36c777a 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/StringMax.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/StringMax.java
@@ -22,6 +22,7 @@ import org.apache.accumulo.core.data.Value;
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.user.MaxCombiner} with
* {@link org.apache.accumulo.core.iterators.LongCombiner.Type#STRING}
*/
+@Deprecated
public class StringMax implements Aggregator {
long max = Long.MIN_VALUE;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/StringMin.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/StringMin.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/StringMin.java
index a712c2c..1e6b7c7 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/StringMin.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/StringMin.java
@@ -22,6 +22,7 @@ import org.apache.accumulo.core.data.Value;
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.user.MinCombiner} with
* {@link org.apache.accumulo.core.iterators.LongCombiner.Type#STRING}
*/
+@Deprecated
public class StringMin implements Aggregator {
long min = Long.MAX_VALUE;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/StringSummation.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/StringSummation.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/StringSummation.java
index b2f8520..00bfd10 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/StringSummation.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/StringSummation.java
@@ -22,6 +22,7 @@ import org.apache.accumulo.core.data.Value;
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.user.SummingCombiner} with
* {@link org.apache.accumulo.core.iterators.LongCombiner.Type#STRING}
*/
+@Deprecated
public class StringSummation implements Aggregator {
long sum = 0;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/conf/AggregatorConfiguration.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/conf/AggregatorConfiguration.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/conf/AggregatorConfiguration.java
index 403df09..a74dd6d 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/conf/AggregatorConfiguration.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/conf/AggregatorConfiguration.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.io.Text;
/**
* @deprecated since 1.4
*/
+@Deprecated
public class AggregatorConfiguration extends PerColumnIteratorConfig {
public AggregatorConfiguration(Text columnFamily, String aggClassName) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/conf/AggregatorSet.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/conf/AggregatorSet.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/conf/AggregatorSet.java
index 9e9e7e1..afa7587 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/conf/AggregatorSet.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/aggregation/conf/AggregatorSet.java
@@ -25,6 +25,7 @@ import org.apache.accumulo.core.iterators.conf.ColumnToClassMapping;
/**
* @deprecated since 1.4
*/
+@Deprecated
public class AggregatorSet extends ColumnToClassMapping<Aggregator> {
public AggregatorSet(Map<String,String> opts) throws InstantiationException, IllegalAccessException, ClassNotFoundException {
super(opts, Aggregator.class);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/conf/PerColumnIteratorConfig.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/conf/PerColumnIteratorConfig.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/conf/PerColumnIteratorConfig.java
index 6b90d22..1d8a1b3 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/conf/PerColumnIteratorConfig.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/conf/PerColumnIteratorConfig.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.io.Text;
* @see org.apache.accumulo.core.client.IteratorSetting.Column
* @see org.apache.accumulo.core.iterators.Combiner#setColumns(org.apache.accumulo.core.client.IteratorSetting, java.util.List)
*/
+@Deprecated
public class PerColumnIteratorConfig {
private String parameter;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/AgeOffFilter.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/AgeOffFilter.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/AgeOffFilter.java
index e9311f4..feea5c3 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/AgeOffFilter.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/AgeOffFilter.java
@@ -26,6 +26,7 @@ import org.apache.accumulo.core.iterators.OptionDescriber;
/**
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.user.AgeOffFilter}
**/
+@Deprecated
public class AgeOffFilter implements Filter, OptionDescriber {
private long threshold;
private long currentTime;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/ColumnAgeOffFilter.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/ColumnAgeOffFilter.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/ColumnAgeOffFilter.java
index a20334b..de9a82a 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/ColumnAgeOffFilter.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/ColumnAgeOffFilter.java
@@ -29,6 +29,7 @@ import org.apache.accumulo.core.iterators.conf.PerColumnIteratorConfig;
/**
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.user.ColumnAgeOffFilter}
**/
+@Deprecated
public class ColumnAgeOffFilter implements Filter, OptionDescriber {
private class TTLSet extends ColumnToClassMapping<Long> {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/ColumnQualifierFilter.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/ColumnQualifierFilter.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/ColumnQualifierFilter.java
index 0b46399..3482c4b 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/ColumnQualifierFilter.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/ColumnQualifierFilter.java
@@ -30,6 +30,7 @@ import org.apache.accumulo.core.data.Value;
/**
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.system.ColumnQualifierFilter}
**/
+@Deprecated
public class ColumnQualifierFilter implements Filter {
private boolean scanColumns;
private HashSet<ByteSequence> columnFamilies;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/DeleteFilter.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/DeleteFilter.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/DeleteFilter.java
index 4c84006..ee9fc13 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/DeleteFilter.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/DeleteFilter.java
@@ -24,6 +24,7 @@ import org.apache.accumulo.core.data.Value;
*
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.Filter} with negate flag set to <tt>true</tt>
**/
+@Deprecated
public class DeleteFilter extends RegExFilter {
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/Filter.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/Filter.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/Filter.java
index 5f458f0..81dd16c 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/Filter.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/Filter.java
@@ -24,6 +24,7 @@ import org.apache.accumulo.core.data.Value;
/**
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.Filter}
**/
+@Deprecated
public interface Filter {
public void init(Map<String,String> options);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/NoLabelFilter.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/NoLabelFilter.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/NoLabelFilter.java
index 9726472..2ef88c4 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/NoLabelFilter.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/NoLabelFilter.java
@@ -26,6 +26,7 @@ import org.apache.accumulo.core.security.ColumnVisibility;
/**
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.user.ReqVisFilter}
**/
+@Deprecated
public class NoLabelFilter implements Filter, OptionDescriber {
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/RegExFilter.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/RegExFilter.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/RegExFilter.java
index f876829..26f5d83 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/RegExFilter.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/RegExFilter.java
@@ -30,6 +30,7 @@ import org.apache.accumulo.core.util.ByteArrayBackedCharSequence;
/**
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.user.RegExFilter}
**/
+@Deprecated
public class RegExFilter implements Filter, OptionDescriber {
public static final String ROW_REGEX = "rowRegex";
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/VisibilityFilter.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/VisibilityFilter.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/VisibilityFilter.java
index 47ef1cc..e47162b 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/VisibilityFilter.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/filter/VisibilityFilter.java
@@ -31,6 +31,7 @@ import org.apache.log4j.Logger;
/**
* @deprecated since 1.4, replaced by {@link org.apache.accumulo.core.iterators.system.VisibilityFilter}
**/
+@Deprecated
public class VisibilityFilter implements Filter {
private VisibilityEvaluator ve;
private Text defaultVisibility;
[2/5] ACCUMULO-1961 Fix warnings
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3a88955/src/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java b/src/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
index 058973b..33082ea 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
@@ -213,20 +213,20 @@ enum ScanRunState {
public class TabletServer extends AbstractMetricsImpl implements org.apache.accumulo.server.tabletserver.metrics.TabletServerMBean {
private static final Logger log = Logger.getLogger(TabletServer.class);
-
+
private static HashMap<String,Long> prevGcTime = new HashMap<String,Long>();
private static long lastMemorySize = 0;
private static long gcTimeIncreasedCount;
private static AtomicLong scanCount = new AtomicLong();
private static final Class<? extends LoggerStrategy> DEFAULT_LOGGER_STRATEGY = RoundRobinLoggerStrategy.class;
-
+
private static final long MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS = 1000;
-
+
private TabletServerLogger logger;
private LoggerStrategy loggerStrategy;
-
+
protected TabletServerMinCMetrics mincMetrics = new TabletServerMinCMetrics();
-
+
public TabletServer() {
super();
watcher = new TransactionWatcher();
@@ -246,36 +246,36 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
}, 5000, 5000);
}
-
+
private synchronized static void logGCInfo() {
List<GarbageCollectorMXBean> gcmBeans = ManagementFactory.getGarbageCollectorMXBeans();
Runtime rt = Runtime.getRuntime();
-
+
StringBuilder sb = new StringBuilder("gc");
-
+
boolean sawChange = false;
-
+
long maxIncreaseInCollectionTime = 0;
-
+
for (GarbageCollectorMXBean gcBean : gcmBeans) {
Long prevTime = prevGcTime.get(gcBean.getName());
long pt = 0;
if (prevTime != null) {
pt = prevTime;
}
-
+
long time = gcBean.getCollectionTime();
-
+
if (time - pt != 0) {
sawChange = true;
}
-
+
long increaseInCollectionTime = time - pt;
sb.append(String.format(" %s=%,.2f(+%,.2f) secs", gcBean.getName(), time / 1000.0, increaseInCollectionTime / 1000.0));
maxIncreaseInCollectionTime = Math.max(increaseInCollectionTime, maxIncreaseInCollectionTime);
prevGcTime.put(gcBean.getName(), time);
}
-
+
long mem = rt.freeMemory();
if (maxIncreaseInCollectionTime == 0) {
gcTimeIncreasedCount = 0;
@@ -286,84 +286,85 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
gcTimeIncreasedCount = 0;
}
}
-
+
if (mem > lastMemorySize) {
sawChange = true;
}
-
+
String sign = "+";
if (mem - lastMemorySize <= 0) {
sign = "";
}
-
+
sb.append(String.format(" freemem=%,d(%s%,d) totalmem=%,d", mem, sign, (mem - lastMemorySize), rt.totalMemory()));
-
+
if (sawChange) {
log.debug(sb.toString());
}
-
+
final long keepAliveTimeout = ServerConfiguration.getSystemConfiguration().getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT);
if (maxIncreaseInCollectionTime > keepAliveTimeout) {
Halt.halt("Garbage collection may be interfering with lock keep-alive. Halting.", -1);
}
-
+
lastMemorySize = mem;
}
-
+
private TabletStatsKeeper statsKeeper;
-
+
private static class Session {
long lastAccessTime;
long startTime;
String user;
String client = TServerUtils.clientAddress.get();
public boolean reserved;
-
+
public void cleanup() {}
}
-
+
private static class SessionManager {
-
+
SecureRandom random;
Map<Long,Session> sessions;
-
+
SessionManager() {
random = new SecureRandom();
sessions = new HashMap<Long,Session>();
-
+
final long maxIdle = ServerConfiguration.getSystemConfiguration().getTimeInMillis(Property.TSERV_SESSION_MAXIDLE);
-
+
TimerTask r = new TimerTask() {
+ @Override
public void run() {
sweep(maxIdle);
}
};
-
+
SimpleTimer.getInstance().schedule(r, 0, Math.max(maxIdle / 2, 1000));
}
-
+
synchronized long createSession(Session session, boolean reserve) {
long sid = random.nextLong();
-
+
while (sessions.containsKey(sid)) {
sid = random.nextLong();
}
-
+
sessions.put(sid, session);
-
+
session.reserved = reserve;
-
+
session.startTime = session.lastAccessTime = System.currentTimeMillis();
-
+
return sid;
}
-
+
/**
* while a session is reserved, it cannot be canceled or removed
*
* @param sessionId
*/
-
+
synchronized Session reserveSession(long sessionId) {
Session session = sessions.get(sessionId);
if (session != null) {
@@ -371,44 +372,44 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
throw new IllegalStateException();
session.reserved = true;
}
-
+
return session;
-
+
}
-
+
synchronized void unreserveSession(Session session) {
if (!session.reserved)
throw new IllegalStateException();
session.reserved = false;
session.lastAccessTime = System.currentTimeMillis();
}
-
+
synchronized void unreserveSession(long sessionId) {
Session session = getSession(sessionId);
if (session != null)
unreserveSession(session);
}
-
+
synchronized Session getSession(long sessionId) {
Session session = sessions.get(sessionId);
if (session != null)
session.lastAccessTime = System.currentTimeMillis();
return session;
}
-
+
Session removeSession(long sessionId) {
Session session = null;
synchronized (this) {
session = sessions.remove(sessionId);
}
-
+
// do clean up out side of lock..
if (session != null)
session.cleanup();
-
+
return session;
}
-
+
private void sweep(long maxIdle) {
ArrayList<Session> sessionsToCleanup = new ArrayList<Session>();
synchronized (this) {
@@ -422,18 +423,19 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
}
}
-
+
// do clean up outside of lock
for (Session session : sessionsToCleanup) {
session.cleanup();
}
}
-
+
synchronized void removeIfNotAccessed(final long sessionId, long delay) {
Session session = sessions.get(sessionId);
if (session != null) {
final long removeTime = session.lastAccessTime;
TimerTask r = new TimerTask() {
+ @Override
public void run() {
Session sessionToCleanup = null;
synchronized (SessionManager.this) {
@@ -443,26 +445,26 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
sessionToCleanup = session2;
}
}
-
+
// call clean up outside of lock
if (sessionToCleanup != null)
sessionToCleanup.cleanup();
}
};
-
+
SimpleTimer.getInstance().schedule(r, delay);
}
}
-
+
public synchronized Map<String,MapCounter<ScanRunState>> getActiveScansPerTable() {
Map<String,MapCounter<ScanRunState>> counts = new HashMap<String,MapCounter<ScanRunState>>();
for (Entry<Long,Session> entry : sessions.entrySet()) {
-
+
Session session = entry.getValue();
@SuppressWarnings("rawtypes")
ScanTask nbt = null;
String tableID = null;
-
+
if (session instanceof ScanSession) {
ScanSession ss = (ScanSession) session;
nbt = ss.nextBatchTask;
@@ -472,40 +474,40 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
nbt = mss.lookupTask;
tableID = mss.threadPoolExtent.getTableId().toString();
}
-
+
if (nbt == null)
continue;
-
+
ScanRunState srs = nbt.getScanRunState();
-
+
if (nbt == null || srs == ScanRunState.FINISHED)
continue;
-
+
MapCounter<ScanRunState> stateCounts = counts.get(tableID);
if (stateCounts == null) {
stateCounts = new MapCounter<ScanRunState>();
counts.put(tableID, stateCounts);
}
-
+
stateCounts.increment(srs, 1);
}
-
+
return counts;
}
-
+
public synchronized List<ActiveScan> getActiveScans() {
-
+
ArrayList<ActiveScan> activeScans = new ArrayList<ActiveScan>();
-
+
long ct = System.currentTimeMillis();
-
+
for (Entry<Long,Session> entry : sessions.entrySet()) {
Session session = entry.getValue();
if (session instanceof ScanSession) {
ScanSession ss = (ScanSession) session;
-
+
ScanState state = ScanState.RUNNING;
-
+
ScanTask<ScanBatch> nbt = ss.nextBatchTask;
if (nbt == null) {
state = ScanState.IDLE;
@@ -517,17 +519,19 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
case FINISHED:
state = ScanState.IDLE;
break;
+ default:
+ break;
}
}
-
+
activeScans.add(new ActiveScan(ss.client, ss.user, ss.extent.getTableId().toString(), ct - ss.startTime, ct - ss.lastAccessTime, ScanType.SINGLE,
state, ss.extent.toThrift(), Translator.translate(ss.columnSet, Translator.CT), ss.ssiList, ss.ssio));
-
+
} else if (session instanceof MultiScanSession) {
MultiScanSession mss = (MultiScanSession) session;
-
+
ScanState state = ScanState.RUNNING;
-
+
ScanTask<MultiScanResult> nbt = mss.lookupTask;
if (nbt == null) {
state = ScanState.IDLE;
@@ -539,44 +543,46 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
case FINISHED:
state = ScanState.IDLE;
break;
+ default:
+ break;
}
}
-
+
activeScans.add(new ActiveScan(mss.client, mss.user, mss.threadPoolExtent.getTableId().toString(), ct - mss.startTime, ct - mss.lastAccessTime,
ScanType.BATCH, state, mss.threadPoolExtent.toThrift(), Translator.translate(mss.columnSet, Translator.CT), mss.ssiList, mss.ssio));
}
}
-
+
return activeScans;
}
}
-
+
static class TservConstraintEnv implements Environment {
-
+
private AuthInfo credentials;
private Authenticator authenticator;
private Authorizations auths;
private KeyExtent ke;
-
+
TservConstraintEnv(Authenticator authenticator, AuthInfo credentials) {
this.authenticator = authenticator;
this.credentials = credentials;
}
-
+
void setExtent(KeyExtent ke) {
this.ke = ke;
}
-
+
@Override
public KeyExtent getExtent() {
return ke;
}
-
+
@Override
public String getUser() {
return credentials.user;
}
-
+
@Override
public Authorizations getAuthorizations() {
if (auths == null)
@@ -587,107 +593,107 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
return auths;
}
-
+
}
-
+
private abstract class ScanTask<T> implements RunnableFuture<T> {
-
+
protected AtomicBoolean interruptFlag;
protected ArrayBlockingQueue<Object> resultQueue;
protected AtomicInteger state;
protected AtomicReference<ScanRunState> runState;
-
+
private static final int INITIAL = 1;
private static final int ADDED = 2;
private static final int CANCELED = 3;
-
+
ScanTask() {
interruptFlag = new AtomicBoolean(false);
runState = new AtomicReference<ScanRunState>(ScanRunState.QUEUED);
state = new AtomicInteger(INITIAL);
resultQueue = new ArrayBlockingQueue<Object>(1);
}
-
+
protected void addResult(Object o) {
if (state.compareAndSet(INITIAL, ADDED))
resultQueue.add(o);
else if (state.get() == ADDED)
throw new IllegalStateException("Tried to add more than one result");
}
-
+
@Override
public boolean cancel(boolean mayInterruptIfRunning) {
if (!mayInterruptIfRunning)
throw new IllegalArgumentException("Cancel will always attempt to interupt running next batch task");
-
+
if (state.get() == CANCELED)
return true;
-
+
if (state.compareAndSet(INITIAL, CANCELED)) {
interruptFlag.set(true);
resultQueue = null;
return true;
}
-
+
return false;
}
-
+
@Override
public T get() throws InterruptedException, ExecutionException {
throw new UnsupportedOperationException();
}
-
+
@SuppressWarnings("unchecked")
@Override
public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
-
+
ArrayBlockingQueue<Object> localRQ = resultQueue;
-
+
if (state.get() == CANCELED)
throw new CancellationException();
-
+
if (localRQ == null && state.get() == ADDED)
throw new IllegalStateException("Tried to get result twice");
-
+
Object r = localRQ.poll(timeout, unit);
-
+
// could have been canceled while waiting
if (state.get() == CANCELED) {
if (r != null)
throw new IllegalStateException("Nothing should have been added when in canceled state");
-
+
throw new CancellationException();
}
-
+
if (r == null)
throw new TimeoutException();
-
+
// make this method stop working now that something is being
// returned
resultQueue = null;
-
+
if (r instanceof Throwable)
throw new ExecutionException((Throwable) r);
-
+
return (T) r;
}
-
+
@Override
public boolean isCancelled() {
return state.get() == CANCELED;
}
-
+
@Override
public boolean isDone() {
return runState.get().equals(ScanRunState.FINISHED);
}
-
+
public ScanRunState getScanRunState() {
return runState.get();
}
-
+
}
-
+
private static class UpdateSession extends Session {
public Tablet currentTablet;
public MapCounter<Tablet> successfulCommits = new MapCounter<Tablet>();
@@ -705,7 +711,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
public long queuedMutationSize = 0;
TservConstraintEnv cenv = null;
}
-
+
private static class ScanSession extends Session {
public KeyExtent extent;
public HashSet<Column> columnSet;
@@ -717,7 +723,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
public volatile ScanTask<ScanBatch> nextBatchTask;
public AtomicBoolean interruptFlag;
public Scanner scanner;
-
+
@Override
public void cleanup() {
try {
@@ -728,32 +734,32 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
scanner.close();
}
}
-
+
}
-
+
private static class MultiScanSession extends Session {
HashSet<Column> columnSet;
Map<KeyExtent,List<Range>> queries;
public List<IterInfo> ssiList;
public Map<String,Map<String,String>> ssio;
public Authorizations auths;
-
+
// stats
int numRanges;
int numTablets;
int numEntries;
long totalLookupTime;
-
+
public volatile ScanTask<MultiScanResult> lookupTask;
public KeyExtent threadPoolExtent;
-
+
@Override
public void cleanup() {
if (lookupTask != null)
lookupTask.cancel(true);
}
}
-
+
/**
* This little class keeps track of writes in progress and allows readers to wait for writes that started before the read. It assumes that the operation ids
* are monotonically increasing.
@@ -762,38 +768,38 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
static class WriteTracker {
private static AtomicLong operationCounter = new AtomicLong(1);
private Map<TabletType,TreeSet<Long>> inProgressWrites = new EnumMap<TabletType,TreeSet<Long>>(TabletType.class);
-
+
WriteTracker() {
for (TabletType ttype : TabletType.values()) {
inProgressWrites.put(ttype, new TreeSet<Long>());
}
}
-
+
synchronized long startWrite(TabletType ttype) {
long operationId = operationCounter.getAndIncrement();
inProgressWrites.get(ttype).add(operationId);
return operationId;
}
-
+
synchronized void finishWrite(long operationId) {
if (operationId == -1)
return;
-
+
boolean removed = false;
-
+
for (TabletType ttype : TabletType.values()) {
removed = inProgressWrites.get(ttype).remove(operationId);
if (removed)
break;
}
-
+
if (!removed) {
throw new IllegalArgumentException("Attempted to finish write not in progress, operationId " + operationId);
}
-
+
this.notifyAll();
}
-
+
synchronized void waitForWrites(TabletType ttype) {
long operationId = operationCounter.getAndIncrement();
while (inProgressWrites.get(ttype).floor(operationId) != null) {
@@ -804,34 +810,34 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
}
}
-
+
public long startWrite(Set<Tablet> keySet) {
if (keySet.size() == 0)
return -1;
-
+
ArrayList<KeyExtent> extents = new ArrayList<KeyExtent>(keySet.size());
-
+
for (Tablet tablet : keySet)
extents.add(tablet.getExtent());
-
+
return startWrite(TabletType.type(extents));
}
}
-
+
TransactionWatcher watcher;
-
+
private class ThriftClientHandler extends ClientServiceHandler implements TabletClientService.Iface {
-
+
SessionManager sessionManager;
-
+
AccumuloConfiguration acuConf = ServerConfiguration.getSystemConfiguration();
-
+
TabletServerUpdateMetrics updateMetrics = new TabletServerUpdateMetrics();
-
+
TabletServerScanMetrics scanMetrics = new TabletServerScanMetrics();
-
+
WriteTracker writeTracker = new WriteTracker();
-
+
ThriftClientHandler() {
super(watcher);
log.debug(ThriftClientHandler.class.getName() + " created");
@@ -844,26 +850,26 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
log.error("Exception registering MBean with MBean Server", e);
}
}
-
+
@Override
public List<TKeyExtent> bulkImport(TInfo tinfo, AuthInfo credentials, long tid, Map<TKeyExtent,Map<String,MapFileInfo>> files, boolean setTime)
throws ThriftSecurityException {
-
+
try {
if (!authenticator.hasSystemPermission(credentials, credentials.user, SystemPermission.SYSTEM))
throw new ThriftSecurityException(credentials.user, SecurityErrorCode.PERMISSION_DENIED);
} catch (AccumuloSecurityException e) {
throw e.asThriftException();
}
-
+
List<TKeyExtent> failures = new ArrayList<TKeyExtent>();
-
+
for (Entry<TKeyExtent,Map<String,MapFileInfo>> entry : files.entrySet()) {
TKeyExtent tke = entry.getKey();
Map<String,MapFileInfo> fileMap = entry.getValue();
-
+
Tablet importTablet = onlineTablets.get(new KeyExtent(tke));
-
+
if (importTablet == null) {
failures.add(tke);
} else {
@@ -877,46 +883,46 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
return failures;
}
-
+
private class NextBatchTask extends ScanTask<ScanBatch> {
-
+
private long scanID;
-
+
NextBatchTask(long scanID, AtomicBoolean interruptFlag) {
this.scanID = scanID;
this.interruptFlag = interruptFlag;
-
+
if (interruptFlag.get())
cancel(true);
}
-
+
@Override
public void run() {
-
+
ScanSession scanSession = (ScanSession) sessionManager.getSession(scanID);
String oldThreadName = Thread.currentThread().getName();
try {
if (isCancelled() || scanSession == null)
return;
-
+
Thread.currentThread().setName(
"User: " + scanSession.user + " Start: " + scanSession.startTime + " Client: " + scanSession.client + " Tablet: " + scanSession.extent);
runState.set(ScanRunState.RUNNING);
-
+
Tablet tablet = onlineTablets.get(scanSession.extent);
-
+
if (tablet == null) {
addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
return;
}
-
+
long t1 = System.currentTimeMillis();
ScanBatch batch = scanSession.scanner.read();
long t2 = System.currentTimeMillis();
scanSession.nbTimes.addStat(t2 - t1);
-
+
// there should only be one thing on the queue at a time, so
// it should be ok to call add()
// instead of put()... if add() fails because queue is at
@@ -939,23 +945,23 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
runState.set(ScanRunState.FINISHED);
Thread.currentThread().setName(oldThreadName);
}
-
+
}
}
-
+
private class LookupTask extends ScanTask<MultiScanResult> {
-
+
private long scanID;
-
+
LookupTask(long scanID) {
this.scanID = scanID;
}
-
+
@Override
public void run() {
MultiScanSession session = (MultiScanSession) sessionManager.getSession(scanID);
String oldThreadName = Thread.currentThread().getName();
-
+
try {
if (isCancelled() || session == null)
return;
@@ -966,24 +972,24 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
long maxResultsSize = acuConf.getMemoryInBytes(Property.TABLE_SCAN_MAXMEM);
long bytesAdded = 0;
long maxScanTime = 4000;
-
+
long startTime = System.currentTimeMillis();
-
+
ArrayList<KVEntry> results = new ArrayList<KVEntry>();
Map<KeyExtent,List<Range>> failures = new HashMap<KeyExtent,List<Range>>();
ArrayList<KeyExtent> fullScans = new ArrayList<KeyExtent>();
KeyExtent partScan = null;
Key partNextKey = null;
boolean partNextKeyInclusive = false;
-
+
Iterator<Entry<KeyExtent,List<Range>>> iter = session.queries.entrySet().iterator();
-
+
// check the time so that the read ahead thread is not monopolized
while (iter.hasNext() && bytesAdded < maxResultsSize && (System.currentTimeMillis() - startTime) < maxScanTime) {
Entry<KeyExtent,List<Range>> entry = iter.next();
-
+
iter.remove();
-
+
// check that tablet server is serving requested tablet
Tablet tablet = onlineTablets.get(entry.getKey());
if (tablet == null) {
@@ -992,32 +998,32 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
Thread.currentThread().setName(
"Client: " + session.client + " User: " + session.user + " Start: " + session.startTime + " Tablet: " + entry.getKey().toString());
-
+
LookupResult lookupResult;
try {
-
+
// do the following check to avoid a race condition
// between setting false below and the task being
// canceled
if (isCancelled())
interruptFlag.set(true);
-
+
lookupResult = tablet.lookup(entry.getValue(), session.columnSet, session.auths, results, maxResultsSize - bytesAdded, session.ssiList,
session.ssio, interruptFlag);
-
+
// if the tablet was closed it it possible that the
// interrupt flag was set.... do not want it set for
// the next
// lookup
interruptFlag.set(false);
-
+
} catch (IOException e) {
log.warn("lookup failed for tablet " + entry.getKey(), e);
throw new RuntimeException(e);
}
-
+
bytesAdded += lookupResult.bytesAdded;
-
+
if (lookupResult.unfinishedRanges.size() > 0) {
if (lookupResult.closed) {
failures.put(entry.getKey(), lookupResult.unfinishedRanges);
@@ -1031,11 +1037,11 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
fullScans.add(entry.getKey());
}
}
-
+
long finishTime = System.currentTimeMillis();
session.totalLookupTime += (finishTime - startTime);
session.numEntries += results.size();
-
+
// convert everything to thrift before adding result
List<TKeyValue> retResults = new ArrayList<TKeyValue>();
for (KVEntry entry : results)
@@ -1064,18 +1070,18 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
}
}
-
+
@Override
public InitialScan startScan(TInfo tinfo, AuthInfo credentials, TKeyExtent textent, TRange range, List<TColumn> columns, int batchSize,
List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites, boolean isolated)
throws NotServingTabletException, ThriftSecurityException, org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
-
+
Authorizations userauths = null;
-
+
try {
if (!authenticator.hasTablePermission(credentials, credentials.user, new String(textent.getTable()), TablePermission.READ))
throw new ThriftSecurityException(credentials.user, SecurityErrorCode.PERMISSION_DENIED);
-
+
userauths = authenticator.getUserAuthorizations(credentials, credentials.user);
for (ByteBuffer auth : authorizations)
if (!userauths.contains(ByteBufferUtil.toBytes(auth)))
@@ -1083,11 +1089,11 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
} catch (AccumuloSecurityException e) {
throw e.asThriftException();
}
-
+
scanCount.addAndGet(1);
-
+
KeyExtent extent = new KeyExtent(textent);
-
+
// wait for any writes that are in flight.. this done to ensure
// consistency across client restarts... assume a client writes
// to accumulo and dies while waiting for a confirmation from
@@ -1100,11 +1106,11 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
// !METADATA table
if (waitForWrites)
writeTracker.waitForWrites(TabletType.type(extent));
-
+
Tablet tablet = onlineTablets.get(extent);
if (tablet == null)
throw new NotServingTabletException(textent);
-
+
ScanSession scanSession = new ScanSession();
scanSession.user = credentials.user;
scanSession.extent = new KeyExtent(extent);
@@ -1112,16 +1118,16 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
scanSession.ssiList = ssiList;
scanSession.ssio = ssio;
scanSession.interruptFlag = new AtomicBoolean();
-
+
for (TColumn tcolumn : columns) {
scanSession.columnSet.add(new Column(tcolumn));
}
-
+
scanSession.scanner = tablet.createScanner(new Range(range), batchSize, scanSession.columnSet, new Authorizations(authorizations), ssiList, ssio,
isolated, scanSession.interruptFlag);
-
+
long sid = sessionManager.createSession(scanSession, true);
-
+
ScanResult scanResult;
try {
scanResult = continueScan(tinfo, sid, scanSession);
@@ -1131,10 +1137,10 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
} finally {
sessionManager.unreserveSession(sid);
}
-
+
return new InitialScan(sid, scanResult);
}
-
+
@Override
public ScanResult continueScan(TInfo tinfo, long scanID) throws NoSuchScanIDException, NotServingTabletException,
org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
@@ -1142,22 +1148,22 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
if (scanSession == null) {
throw new NoSuchScanIDException();
}
-
+
try {
return continueScan(tinfo, scanID, scanSession);
} finally {
sessionManager.unreserveSession(scanSession);
}
}
-
+
private ScanResult continueScan(TInfo tinfo, long scanID, ScanSession scanSession) throws NoSuchScanIDException, NotServingTabletException,
org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
-
+
if (scanSession.nextBatchTask == null) {
scanSession.nextBatchTask = new NextBatchTask(scanID, scanSession.interruptFlag);
resourceManager.executeReadAhead(scanSession.extent, scanSession.nextBatchTask);
}
-
+
ScanBatch bresult;
try {
bresult = scanSession.nextBatchTask.get(MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS, TimeUnit.MILLISECONDS);
@@ -1187,32 +1193,32 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
log.warn("Failed to get next batch", t);
throw new RuntimeException(t);
}
-
+
ScanResult scanResult = new ScanResult(Key.compress(bresult.results), bresult.more);
-
+
scanSession.entriesReturned += scanResult.results.size();
-
+
scanSession.batchCount++;
-
+
if (scanResult.more && scanSession.batchCount > 3) {
// start reading next batch while current batch is transmitted
// to client
scanSession.nextBatchTask = new NextBatchTask(scanID, scanSession.interruptFlag);
resourceManager.executeReadAhead(scanSession.extent, scanSession.nextBatchTask);
}
-
+
if (!scanResult.more)
closeScan(tinfo, scanID);
-
+
return scanResult;
}
-
+
@Override
public void closeScan(TInfo tinfo, long scanID) {
ScanSession ss = (ScanSession) sessionManager.removeSession(scanID);
if (ss != null) {
long t2 = System.currentTimeMillis();
-
+
log.debug(String.format("ScanSess tid %s %s %,d entries in %.2f secs, nbTimes = [%s] ", TServerUtils.clientAddress.get(), ss.extent.getTableId()
.toString(), ss.entriesReturned, (t2 - ss.startTime) / 1000.0, ss.nbTimes.toString()));
if (scanMetrics.isEnabled()) {
@@ -1221,7 +1227,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
}
}
-
+
@Override
public InitialMultiScan startMultiScan(TInfo tinfo, AuthInfo credentials, Map<TKeyExtent,List<TRange>> tbatch, List<TColumn> tcolumns,
List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites) throws ThriftSecurityException {
@@ -1230,14 +1236,14 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
for (TKeyExtent keyExtent : tbatch.keySet()) {
tables.add(new String(keyExtent.getTable()));
}
-
+
// check if user has permission to the tables
Authorizations userauths = null;
try {
for (String table : tables)
if (!authenticator.hasTablePermission(credentials, credentials.user, table, TablePermission.READ))
throw new ThriftSecurityException(credentials.user, SecurityErrorCode.PERMISSION_DENIED);
-
+
userauths = authenticator.getUserAuthorizations(credentials, credentials.user);
for (ByteBuffer auth : authorizations)
if (!userauths.contains(ByteBufferUtil.toBytes(auth)))
@@ -1245,11 +1251,11 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
} catch (AccumuloSecurityException e) {
throw e.asThriftException();
}
-
+
KeyExtent threadPoolExtent = null;
-
+
Map<KeyExtent,List<Range>> batch = Translator.translate(tbatch, Translator.TKET, new Translator.ListTranslator<TRange,Range>(Translator.TRT));
-
+
for (KeyExtent keyExtent : batch.keySet()) {
if (threadPoolExtent == null) {
threadPoolExtent = keyExtent;
@@ -1259,12 +1265,12 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
&& !threadPoolExtent.getTableId().toString().equals(Constants.METADATA_TABLE_ID)) {
throw new IllegalArgumentException("Cannot batch query !METADATA and non !METADATA tablets " + threadPoolExtent + " " + keyExtent);
}
-
+
}
-
+
if (waitForWrites)
writeTracker.waitForWrites(TabletType.type(batch.keySet()));
-
+
MultiScanSession mss = new MultiScanSession();
mss.user = credentials.user;
mss.queries = batch;
@@ -1272,19 +1278,19 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
mss.ssiList = ssiList;
mss.ssio = ssio;
mss.auths = new Authorizations(authorizations);
-
+
mss.numTablets = batch.size();
for (List<Range> ranges : batch.values()) {
mss.numRanges += ranges.size();
}
-
+
for (TColumn tcolumn : tcolumns)
mss.columnSet.add(new Column(tcolumn));
-
+
mss.threadPoolExtent = threadPoolExtent;
-
+
long sid = sessionManager.createSession(mss, true);
-
+
MultiScanResult result;
try {
result = continueMultiScan(tinfo, sid, mss);
@@ -1294,34 +1300,34 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
} finally {
sessionManager.unreserveSession(sid);
}
-
+
scanCount.addAndGet(batch.size());
return new InitialMultiScan(sid, result);
}
-
+
@Override
public MultiScanResult continueMultiScan(TInfo tinfo, long scanID) throws NoSuchScanIDException {
-
+
MultiScanSession session = (MultiScanSession) sessionManager.reserveSession(scanID);
-
+
if (session == null) {
throw new NoSuchScanIDException();
}
-
+
try {
return continueMultiScan(tinfo, scanID, session);
} finally {
sessionManager.unreserveSession(session);
}
}
-
+
private MultiScanResult continueMultiScan(TInfo tinfo, long scanID, MultiScanSession session) throws NoSuchScanIDException {
-
+
if (session.lookupTask == null) {
session.lookupTask = new LookupTask(scanID);
resourceManager.executeReadAhead(session.threadPoolExtent, session.lookupTask);
}
-
+
try {
MultiScanResult scanResult = session.lookupTask.get(MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS, TimeUnit.MILLISECONDS);
session.lookupTask = null;
@@ -1339,19 +1345,19 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
throw new RuntimeException(t);
}
}
-
+
@Override
public void closeMultiScan(TInfo tinfo, long scanID) throws NoSuchScanIDException {
MultiScanSession session = (MultiScanSession) sessionManager.removeSession(scanID);
if (session == null) {
throw new NoSuchScanIDException();
}
-
+
long t2 = System.currentTimeMillis();
log.debug(String.format("MultiScanSess %s %,d entries in %.2f secs (lookup_time:%.2f secs tablets:%,d ranges:%,d) ", TServerUtils.clientAddress.get(),
session.numEntries, (t2 - session.startTime) / 1000.0, session.totalLookupTime / 1000.0, session.numTablets, session.numRanges));
}
-
+
@Override
public long startUpdate(TInfo tinfo, AuthInfo credentials) throws ThriftSecurityException {
// Make sure user is real
@@ -1364,17 +1370,17 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
} catch (AccumuloSecurityException e) {
throw e.asThriftException();
}
-
+
UpdateSession us = new UpdateSession();
us.violations = new Violations();
us.credentials = credentials;
us.cenv = new TservConstraintEnv(authenticator, credentials);
-
+
long sid = sessionManager.createSession(us, false);
-
+
return sid;
}
-
+
private void setUpdateTablet(UpdateSession us, KeyExtent keyExtent) {
long t1 = System.currentTimeMillis();
if (us.currentTablet != null && us.currentTablet.getExtent().equals(keyExtent))
@@ -1384,7 +1390,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
// if there were previous failures, then do not accept additional writes
return;
}
-
+
try {
// if user has no permission to write to this table, add it to
// the failures list
@@ -1425,18 +1431,18 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
return;
}
}
-
+
@Override
public void applyUpdates(TInfo tinfo, long updateID, TKeyExtent tkeyExtent, List<TMutation> tmutations) {
UpdateSession us = (UpdateSession) sessionManager.reserveSession(updateID);
if (us == null) {
throw new RuntimeException("No Such SessionID");
}
-
+
try {
KeyExtent keyExtent = new KeyExtent(tkeyExtent);
setUpdateTablet(us, keyExtent);
-
+
if (us.currentTablet != null) {
List<Mutation> mutations = us.queuedMutations.get(us.currentTablet);
for (TMutation tmutation : tmutations) {
@@ -1451,33 +1457,33 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
sessionManager.unreserveSession(us);
}
}
-
+
private void flush(UpdateSession us) {
-
+
int mutationCount = 0;
Map<CommitSession,List<Mutation>> sendables = new HashMap<CommitSession,List<Mutation>>();
Throwable error = null;
-
+
long pt1 = System.currentTimeMillis();
-
+
boolean containsMetadataTablet = false;
for (Tablet tablet : us.queuedMutations.keySet())
if (tablet.getExtent().getTableId().toString().equals(Constants.METADATA_TABLE_ID))
containsMetadataTablet = true;
-
+
if (!containsMetadataTablet && us.queuedMutations.size() > 0)
TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
-
+
Span prep = Trace.start("prep");
for (Entry<Tablet,? extends List<Mutation>> entry : us.queuedMutations.entrySet()) {
-
+
Tablet tablet = entry.getKey();
List<Mutation> mutations = entry.getValue();
if (mutations.size() > 0) {
try {
if (updateMetrics.isEnabled())
updateMetrics.add(TabletServerUpdateMetrics.mutationArraySize, mutations.size());
-
+
CommitSession commitSession = tablet.prepareMutationsForCommit(us.cenv, mutations);
if (commitSession == null) {
if (us.currentTablet == tablet) {
@@ -1488,12 +1494,12 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
sendables.put(commitSession, mutations);
mutationCount += mutations.size();
}
-
+
} catch (TConstraintViolationException e) {
us.violations.add(e.getViolations());
if (updateMetrics.isEnabled())
updateMetrics.add(TabletServerUpdateMetrics.constraintViolations, 0);
-
+
if (e.getNonViolators().size() > 0) {
// only log and commit mutations if there were some
// that did not
@@ -1502,9 +1508,9 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
// expects
sendables.put(e.getCommitSession(), e.getNonViolators());
}
-
+
mutationCount += mutations.size();
-
+
} catch (HoldTimeoutException t) {
error = t;
log.debug("Giving up on mutations due to a long memory hold time");
@@ -1517,14 +1523,14 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
}
prep.stop();
-
+
Span wal = Trace.start("wal");
long pt2 = System.currentTimeMillis();
long avgPrepareTime = (long) ((pt2 - pt1) / (double) us.queuedMutations.size());
us.prepareTimes.addStat(pt2 - pt1);
if (updateMetrics.isEnabled())
updateMetrics.add(TabletServerUpdateMetrics.commitPrep, (avgPrepareTime));
-
+
if (error != null) {
for (Entry<CommitSession,List<Mutation>> e : sendables.entrySet()) {
e.getKey().abortCommit(e.getValue());
@@ -1535,14 +1541,14 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
while (true) {
try {
long t1 = System.currentTimeMillis();
-
+
logger.logManyTablets(sendables);
-
+
long t2 = System.currentTimeMillis();
us.walogTimes.addStat(t2 - t1);
if (updateMetrics.isEnabled())
updateMetrics.add(TabletServerUpdateMetrics.waLogWriteTime, (t2 - t1));
-
+
break;
} catch (IOException ex) {
log.warn("logging mutations failed, retrying");
@@ -1551,19 +1557,19 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
throw new RuntimeException(t);
}
}
-
+
wal.stop();
-
+
Span commit = Trace.start("commit");
long t1 = System.currentTimeMillis();
for (Entry<CommitSession,? extends List<Mutation>> entry : sendables.entrySet()) {
CommitSession commitSession = entry.getKey();
List<Mutation> mutations = entry.getValue();
-
+
commitSession.commit(mutations);
-
+
Tablet tablet = commitSession.getTablet();
-
+
if (tablet == us.currentTablet) {
// because constraint violations may filter out some
// mutations, for proper
@@ -1575,12 +1581,12 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
}
long t2 = System.currentTimeMillis();
-
+
long avgCommitTime = (long) ((t2 - t1) / (double) sendables.size());
-
+
us.flushTime += (t2 - pt1);
us.commitTimes.addStat(t2 - t1);
-
+
if (updateMetrics.isEnabled())
updateMetrics.add(TabletServerUpdateMetrics.commitTime, avgCommitTime);
commit.stop();
@@ -1593,25 +1599,25 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
us.totalUpdates += mutationCount;
}
-
+
@Override
public UpdateErrors closeUpdate(TInfo tinfo, long updateID) throws NoSuchScanIDException {
UpdateSession us = (UpdateSession) sessionManager.removeSession(updateID);
if (us == null) {
throw new NoSuchScanIDException();
}
-
+
// clients may or may not see data from an update session while
// it is in progress, however when the update session is closed
// want to ensure that reads wait for the write to finish
long opid = writeTracker.startWrite(us.queuedMutations.keySet());
-
+
try {
flush(us);
} finally {
writeTracker.finishWrite(opid);
}
-
+
log.debug(String.format("UpSess %s %,d in %.3fs, at=[%s] ft=%.3fs(pt=%.3fs lt=%.3fs ct=%.3fs)", TServerUtils.clientAddress.get(), us.totalUpdates,
(System.currentTimeMillis() - us.startTime) / 1000.0, us.authTimes.toString(), us.flushTime / 1000.0, us.prepareTimes.getSum() / 1000.0,
us.walogTimes.getSum() / 1000.0, us.commitTimes.getSum() / 1000.0));
@@ -1628,11 +1634,11 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
KeyExtent first = us.authFailures.iterator().next();
log.debug(String.format("Authentication Failures: %d, first %s", us.authFailures.size(), first.toString()));
}
-
+
return new UpdateErrors(Translator.translate(us.failures, Translator.KET), Translator.translate(violations, Translator.CVST), Translator.translate(
us.authFailures, Translator.KET));
}
-
+
@Override
public void update(TInfo tinfo, AuthInfo credentials, TKeyExtent tkeyExtent, TMutation tmutation) throws NotServingTabletException,
ConstraintViolationException, ThriftSecurityException {
@@ -1642,29 +1648,29 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
} catch (AccumuloSecurityException e) {
throw e.asThriftException();
}
-
+
KeyExtent keyExtent = new KeyExtent(tkeyExtent);
Tablet tablet = onlineTablets.get(new KeyExtent(keyExtent));
if (tablet == null) {
throw new NotServingTabletException(tkeyExtent);
}
-
+
if (!keyExtent.getTableId().toString().equals(Constants.METADATA_TABLE_ID))
TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
-
+
long opid = writeTracker.startWrite(TabletType.type(keyExtent));
-
+
try {
Mutation mutation = new Mutation(tmutation);
List<Mutation> mutations = Collections.singletonList(mutation);
-
+
Span prep = Trace.start("prep");
CommitSession cs = tablet.prepareMutationsForCommit(new TservConstraintEnv(authenticator, credentials), mutations);
prep.stop();
if (cs == null) {
throw new NotServingTabletException(tkeyExtent);
}
-
+
while (true) {
try {
Span wal = Trace.start("wal");
@@ -1675,7 +1681,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
log.warn(ex, ex);
}
}
-
+
Span commit = Trace.start("commit");
cs.commit(mutations);
commit.stop();
@@ -1685,7 +1691,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
writeTracker.finishWrite(opid);
}
}
-
+
@Override
public void splitTablet(TInfo tinfo, AuthInfo credentials, TKeyExtent tkeyExtent, ByteBuffer splitPoint) throws NotServingTabletException,
ThriftSecurityException {
@@ -1698,14 +1704,14 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
} catch (AccumuloSecurityException e) {
throw e.asThriftException();
}
-
+
KeyExtent keyExtent = new KeyExtent(tkeyExtent);
-
+
Tablet tablet = onlineTablets.get(keyExtent);
if (tablet == null) {
throw new NotServingTabletException(tkeyExtent);
}
-
+
if (keyExtent.getEndRow() == null || !keyExtent.getEndRow().equals(ByteBufferUtil.toText(splitPoint))) {
try {
if (TabletServer.this.splitTablet(tablet, ByteBufferUtil.toBytes(splitPoint)) == null) {
@@ -1717,12 +1723,12 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
}
}
-
+
@Override
public TabletServerStatus getTabletServerStatus(TInfo tinfo, AuthInfo credentials) throws ThriftSecurityException, TException {
return getStats(sessionManager.getActiveScansPerTable());
}
-
+
@Override
public List<TabletStats> getTabletStats(TInfo tinfo, AuthInfo credentials, String tableId) throws ThriftSecurityException, TException {
TreeMap<KeyExtent,Tablet> onlineTabletsCopy;
@@ -1747,9 +1753,9 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
return result;
}
-
+
private ZooCache masterLockCache = new ZooCache();
-
+
private void checkPermission(AuthInfo credentials, String lock, boolean requiresSystemPermission, final String request) throws ThriftSecurityException {
if (requiresSystemPermission) {
boolean fatal = false;
@@ -1769,6 +1775,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
} finally {
if (fatal) {
Halt.halt(1, new Runnable() {
+ @Override
public void run() {
logGCInfo();
}
@@ -1776,23 +1783,24 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
}
}
-
+
if (tabletServerLock == null || !tabletServerLock.wasLockAcquired()) {
log.warn("Got " + request + " message from master before lock acquired, ignoring...");
throw new RuntimeException("Lock not acquired");
}
-
+
if (tabletServerLock != null && tabletServerLock.wasLockAcquired() && !tabletServerLock.isLocked()) {
Halt.halt(1, new Runnable() {
+ @Override
public void run() {
log.info("Tablet server no longer holds lock during checkPermission() : " + request + ", exiting");
logGCInfo();
}
});
}
-
+
ZooUtil.LockID lid = new ZooUtil.LockID(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZMASTER_LOCK, lock);
-
+
try {
if (!ZooLock.isLockHeld(masterLockCache, lid)) {
// maybe the cache is out of date and a new master holds the
@@ -1807,7 +1815,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
throw new RuntimeException("bad master lock", e);
}
}
-
+
@Override
public void loadTablet(TInfo tinfo, AuthInfo credentials, String lock, final TKeyExtent textent) {
try {
@@ -1816,17 +1824,17 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
log.error(e, e);
throw new RuntimeException(e);
}
-
+
final KeyExtent extent = new KeyExtent(textent);
-
+
synchronized (unopenedTablets) {
synchronized (openingTablets) {
synchronized (onlineTablets) {
-
+
// checking if this exact tablet is in any of the sets
// below is not a strong enough check
// when splits and fix splits occurring
-
+
Set<KeyExtent> unopenedOverlapping = KeyExtent.findOverlapping(extent, unopenedTablets);
Set<KeyExtent> openingOverlapping = KeyExtent.findOverlapping(extent, openingTablets);
Set<KeyExtent> onlineOverlapping = KeyExtent.findOverlapping(extent, onlineTablets);
@@ -1834,26 +1842,27 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
all.addAll(unopenedOverlapping);
all.addAll(openingOverlapping);
all.addAll(onlineOverlapping);
-
+
if (!all.isEmpty()) {
if (all.size() != 1 || !all.contains(extent)) {
log.error("Tablet " + extent + " overlaps previously assigned " + unopenedOverlapping + " " + openingOverlapping + " " + onlineOverlapping);
}
return;
}
-
+
unopenedTablets.add(extent);
}
}
}
-
+
// add the assignment job to the appropriate queue
log.info("Loading tablet " + extent);
-
+
final Runnable ah = new LoggingRunnable(log, new AssignmentHandler(extent));
// Root tablet assignment must take place immediately
if (extent.compareTo(Constants.ROOT_TABLET_EXTENT) == 0) {
new Thread("Root Tablet Assignment") {
+ @Override
public void run() {
ah.run();
if (onlineTablets.containsKey(extent)) {
@@ -1861,7 +1870,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
} else {
log.info("Root tablet failed to load");
}
-
+
}
}.start();
} else {
@@ -1872,7 +1881,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
}
}
-
+
@Override
public void unloadTablet(TInfo tinfo, AuthInfo credentials, String lock, TKeyExtent textent, boolean save) {
try {
@@ -1881,12 +1890,12 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
log.error(e, e);
throw new RuntimeException(e);
}
-
+
KeyExtent extent = new KeyExtent(textent);
-
+
resourceManager.addMigration(extent, new LoggingRunnable(log, new UnloadTabletHandler(extent, save)));
}
-
+
@Override
public void flush(TInfo tinfo, AuthInfo credentials, String lock, String tableId, ByteBuffer startRow, ByteBuffer endRow) {
try {
@@ -1895,19 +1904,19 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
log.error(e, e);
throw new RuntimeException(e);
}
-
+
ArrayList<Tablet> tabletsToFlush = new ArrayList<Tablet>();
-
+
KeyExtent ke = new KeyExtent(new Text(tableId), ByteBufferUtil.toText(endRow), ByteBufferUtil.toText(startRow));
-
+
synchronized (onlineTablets) {
for (Tablet tablet : onlineTablets.values())
if (ke.overlaps(tablet.getExtent()))
tabletsToFlush.add(tablet);
}
-
+
Long flushID = null;
-
+
for (Tablet tablet : tabletsToFlush) {
if (flushID == null) {
// read the flush id once from zookeeper instead of reading
@@ -1923,7 +1932,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
tablet.flush(flushID);
}
}
-
+
@Override
public void flushTablet(TInfo tinfo, AuthInfo credentials, String lock, TKeyExtent textent) throws TException {
try {
@@ -1942,13 +1951,14 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
}
}
-
+
@Override
public void halt(TInfo tinfo, AuthInfo credentials, String lock) throws ThriftSecurityException {
-
+
checkPermission(credentials, lock, true, "halt");
-
+
Halt.halt(0, new Runnable() {
+ @Override
public void run() {
log.info("Master requested tablet server halt");
logGCInfo();
@@ -1961,7 +1971,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
});
}
-
+
@Override
public void fastHalt(TInfo info, AuthInfo credentials, String lock) {
try {
@@ -1970,30 +1980,30 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
log.warn("Error halting", e);
}
}
-
+
@Override
public TabletStats getHistoricalStats(TInfo tinfo, AuthInfo credentials) throws ThriftSecurityException, TException {
return statsKeeper.getTabletStats();
}
-
+
@Override
public void useLoggers(TInfo tinfo, AuthInfo credentials, Set<String> loggers) throws TException {
loggerStrategy.preferLoggers(loggers);
}
-
+
@Override
public List<ActiveScan> getActiveScans(TInfo tinfo, AuthInfo credentials) throws ThriftSecurityException, TException {
-
+
try {
if (!authenticator.hasSystemPermission(credentials, credentials.user, SystemPermission.SYSTEM))
throw new ThriftSecurityException(credentials.user, SecurityErrorCode.PERMISSION_DENIED);
} catch (AccumuloSecurityException e) {
throw e.asThriftException();
}
-
+
return sessionManager.getActiveScans();
}
-
+
@Override
public void chop(TInfo tinfo, AuthInfo credentials, String lock, TKeyExtent textent) throws TException {
try {
@@ -2003,13 +2013,13 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
throw new RuntimeException(e);
}
KeyExtent ke = new KeyExtent(textent);
-
+
Tablet tablet = onlineTablets.get(ke);
if (tablet != null) {
tablet.chopFiles();
}
}
-
+
@Override
public void compact(TInfo tinfo, AuthInfo credentials, String lock, String tableId, ByteBuffer startRow, ByteBuffer endRow) throws TException {
try {
@@ -2018,18 +2028,18 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
} catch (Exception e) {
throw new RuntimeException(e);
}
-
+
KeyExtent ke = new KeyExtent(new Text(tableId), ByteBufferUtil.toText(endRow), ByteBufferUtil.toText(startRow));
-
+
ArrayList<Tablet> tabletsToCompact = new ArrayList<Tablet>();
synchronized (onlineTablets) {
for (Tablet tablet : onlineTablets.values())
if (ke.overlaps(tablet.getExtent()))
tabletsToCompact.add(tablet);
}
-
+
Long compactionId = null;
-
+
for (Tablet tablet : tabletsToCompact) {
// all for the same table id, so only need to read
// compaction id once
@@ -2042,18 +2052,18 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
tablet.compactAll(compactionId);
}
-
+
}
-
+
}
-
+
private class SplitRunner implements Runnable {
private Tablet tablet;
-
+
public SplitRunner(Tablet tablet) {
this.tablet = tablet;
}
-
+
@Override
public void run() {
if (majorCompactorDisabled) {
@@ -2061,33 +2071,34 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
// initiated exit
return;
}
-
+
splitTablet(tablet);
}
}
-
+
boolean isMajorCompactionDisabled() {
return majorCompactorDisabled;
}
-
+
private class MajorCompactor implements Runnable {
private AccumuloConfiguration acuConf = ServerConfiguration.getSystemConfiguration();
-
+
+ @Override
public void run() {
while (!majorCompactorDisabled) {
try {
UtilWaitThread.sleep(acuConf.getTimeInMillis(Property.TSERV_MAJC_DELAY));
-
+
TreeMap<KeyExtent,Tablet> copyOnlineTablets = new TreeMap<KeyExtent,Tablet>();
-
+
synchronized (onlineTablets) {
copyOnlineTablets.putAll(onlineTablets); // avoid
// concurrent
// modification
}
-
+
int numMajorCompactionsInProgress = 0;
-
+
Iterator<Entry<KeyExtent,Tablet>> iter = copyOnlineTablets.entrySet().iterator();
while (iter.hasNext() && !majorCompactorDisabled) { // bail
// early
@@ -2096,26 +2107,26 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
// we're
// shutting
// down
-
+
Entry<KeyExtent,Tablet> entry = iter.next();
-
+
Tablet tablet = entry.getValue();
-
+
// if we need to split AND compact, we need a good way
// to decide what to do
if (tablet.needsSplit()) {
resourceManager.executeSplit(tablet.getExtent(), new LoggingRunnable(log, new SplitRunner(tablet)));
continue;
}
-
+
int maxLogEntriesPerTablet = ServerConfiguration.getTableConfiguration(tablet.getExtent().getTableId().toString()).getCount(
Property.TABLE_MINC_LOGS_MAX);
-
+
if (tablet.getLogCount() >= maxLogEntriesPerTablet) {
log.debug("Initiating minor compaction for " + tablet.getExtent() + " because it has " + tablet.getLogCount() + " write ahead logs");
tablet.initiateMinorCompaction();
}
-
+
synchronized (tablet) {
if (tablet.initiateMajorCompaction(MajorCompactionReason.NORMAL) || tablet.majorCompactionQueued() || tablet.majorCompactionRunning()) {
numMajorCompactionsInProgress++;
@@ -2123,18 +2134,18 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
}
}
-
+
int idleCompactionsToStart = Math.max(1, acuConf.getCount(Property.TSERV_MAJC_MAXCONCURRENT) / 2);
-
+
if (numMajorCompactionsInProgress < idleCompactionsToStart) {
// system is not major compacting, can schedule some
// idle compactions
iter = copyOnlineTablets.entrySet().iterator();
-
+
while (iter.hasNext() && !majorCompactorDisabled && numMajorCompactionsInProgress < idleCompactionsToStart) {
Entry<KeyExtent,Tablet> entry = iter.next();
Tablet tablet = entry.getValue();
-
+
if (tablet.initiateMajorCompaction(MajorCompactionReason.IDLE)) {
numMajorCompactionsInProgress++;
}
@@ -2147,10 +2158,10 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
}
}
-
+
private void splitTablet(Tablet tablet) {
try {
-
+
TreeMap<KeyExtent,SplitInfo> tabletInfo = splitTablet(tablet, null);
if (tabletInfo == null) {
// either split or compact not both
@@ -2166,34 +2177,34 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
log.error("Unknown error on split: " + e, e);
}
}
-
+
private TreeMap<KeyExtent,SplitInfo> splitTablet(Tablet tablet, byte[] splitPoint) throws IOException {
long t1 = System.currentTimeMillis();
-
+
TreeMap<KeyExtent,SplitInfo> tabletInfo = tablet.split(splitPoint);
if (tabletInfo == null) {
return null;
}
-
+
log.info("Starting split: " + tablet.getExtent());
statsKeeper.incrementStatusSplit();
long start = System.currentTimeMillis();
-
+
Tablet[] newTablets = new Tablet[2];
-
+
Entry<KeyExtent,SplitInfo> first = tabletInfo.firstEntry();
newTablets[0] = new Tablet(TabletServer.this, new Text(first.getValue().dir), first.getKey(), resourceManager.createTabletResourceManager(),
first.getValue().datafiles, first.getValue().time, first.getValue().initFlushID, first.getValue().initCompactID);
-
+
Entry<KeyExtent,SplitInfo> last = tabletInfo.lastEntry();
newTablets[1] = new Tablet(TabletServer.this, new Text(last.getValue().dir), last.getKey(), resourceManager.createTabletResourceManager(),
last.getValue().datafiles, last.getValue().time, last.getValue().initFlushID, last.getValue().initCompactID);
-
+
// roll tablet stats over into tablet server's statsKeeper object as
// historical data
statsKeeper.saveMinorTimes(tablet.timer);
statsKeeper.saveMajorTimes(tablet.timer);
-
+
// lose the reference to the old tablet and open two new ones
synchronized (onlineTablets) {
onlineTablets.remove(tablet.getExtent());
@@ -2203,39 +2214,40 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
// tell the master
enqueueMasterMessage(new SplitReportMessage(tablet.getExtent(), newTablets[0].getExtent(), new Text("/" + newTablets[0].getLocation().getName()),
newTablets[1].getExtent(), new Text("/" + newTablets[1].getLocation().getName())));
-
+
statsKeeper.updateTime(Operation.SPLIT, start, 0, false);
long t2 = System.currentTimeMillis();
log.info("Tablet split: " + tablet.getExtent() + " size0 " + newTablets[0].estimateTabletSize() + " size1 " + newTablets[1].estimateTabletSize() + " time "
+ (t2 - t1) + "ms");
-
+
return tabletInfo;
}
-
+
public long lastPingTime = System.currentTimeMillis();
public Socket currentMaster;
-
+
// a queue to hold messages that are to be sent back to the master
private BlockingDeque<MasterMessage> masterMessages = new LinkedBlockingDeque<MasterMessage>();
-
+
// add a message for the main thread to send back to the master
void enqueueMasterMessage(MasterMessage m) {
masterMessages.addLast(m);
}
-
+
private class UnloadTabletHandler implements Runnable {
private KeyExtent extent;
private boolean saveState;
-
+
public UnloadTabletHandler(KeyExtent extent, boolean saveState) {
this.extent = extent;
this.saveState = saveState;
}
-
+
+ @Override
public void run() {
-
+
Tablet t = null;
-
+
synchronized (unopenedTablets) {
if (unopenedTablets.contains(extent)) {
unopenedTablets.remove(extent);
@@ -2255,7 +2267,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
t = onlineTablets.get(extent);
}
}
-
+
if (t == null) {
// Tablet has probably been recently unloaded: repeated master
// unload request is crossing the successful unloaded message
@@ -2265,26 +2277,26 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
return;
}
-
+
try {
t.close(saveState);
} catch (Throwable e) {
-
+
if ((t.isClosing() || t.isClosed()) && e instanceof IllegalStateException) {
log.debug("Failed to unload tablet " + extent + "... it was alread closing or closed : " + e.getMessage());
} else {
log.error("Failed to close tablet " + extent + "... Aborting migration", e);
}
-
+
enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.UNLOAD_ERROR, extent));
return;
}
-
+
// stop serving tablet - client will get not serving tablet
// exceptions
recentlyUnloadedCache.put(extent, System.currentTimeMillis());
onlineTablets.remove(extent);
-
+
try {
TServerInstance instance = new TServerInstance(clientAddress, getLock().getSessionId());
TabletLocationState tls = new TabletLocationState(extent, null, instance, null, null, false);
@@ -2297,38 +2309,39 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
} catch (InterruptedException e) {
log.warn("Interrupted while getting our zookeeper session information", e);
}
-
+
// tell the master how it went
enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.UNLOADED, extent));
-
+
// roll tablet stats over into tablet server's statsKeeper object as
// historical data
statsKeeper.saveMinorTimes(t.timer);
statsKeeper.saveMajorTimes(t.timer);
-
+
log.info("unloaded " + extent);
-
+
}
}
-
+
private class AssignmentHandler implements Runnable {
private KeyExtent extent;
private int retryAttempt = 0;
-
+
public AssignmentHandler(KeyExtent extent) {
this.extent = extent;
}
-
+
public AssignmentHandler(KeyExtent extent, int retryAttempt) {
this(extent);
this.retryAttempt = retryAttempt;
}
-
+
+ @Override
public void run() {
log.info(clientAddress + ": got assignment from master: " + extent);
-
+
final boolean isMetaDataTablet = extent.getTableId().toString().compareTo(Constants.METADATA_TABLE_ID) == 0;
-
+
synchronized (unopenedTablets) {
synchronized (openingTablets) {
synchronized (onlineTablets) {
@@ -2337,23 +2350,23 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
Set<KeyExtent> unopenedOverlapping = KeyExtent.findOverlapping(extent, unopenedTablets);
Set<KeyExtent> openingOverlapping = KeyExtent.findOverlapping(extent, openingTablets);
Set<KeyExtent> onlineOverlapping = KeyExtent.findOverlapping(extent, onlineTablets);
-
+
if (openingOverlapping.contains(extent) || onlineOverlapping.contains(extent))
return;
-
+
if (!unopenedTablets.contains(extent) || unopenedOverlapping.size() != 1 || openingOverlapping.size() > 0 || onlineOverlapping.size() > 0) {
throw new IllegalStateException("overlaps assigned " + extent + " " + !unopenedTablets.contains(extent) + " " + unopenedOverlapping + " "
+ openingOverlapping + " " + onlineOverlapping);
}
}
-
+
unopenedTablets.remove(extent);
openingTablets.add(extent);
}
}
-
+
log.debug("Loading extent: " + extent);
-
+
// check Metadata table before accepting assignment
SortedMap<KeyExtent,Text> tabletsInRange = null;
SortedMap<Key,Value> tabletsKeyValues = new TreeMap<Key,Value>();
@@ -2367,7 +2380,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
log.warn("Failed to verify tablet " + extent, e);
throw new RuntimeException(e);
}
-
+
if (tabletsInRange == null) {
log.info("Reporting tablet " + extent + " assignment failure: unable to verify Tablet Information");
enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.LOAD_FAILURE, extent));
@@ -2379,9 +2392,9 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
// If extent given is not the one to be opened, update
if (tabletsInRange.size() != 1 || !tabletsInRange.containsKey(extent)) {
-
+
tabletsKeyValues.clear();
-
+
synchronized (openingTablets) {
openingTablets.remove(extent);
openingTablets.notifyAll();
@@ -2403,15 +2416,15 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
log.debug("Master didn't know " + extent + " was split, letting it know about " + tabletsInRange.keySet());
enqueueMasterMessage(new SplitReportMessage(extent, tabletsInRange));
}
-
+
// create the tablet object
for (Entry<KeyExtent,Text> entry : tabletsInRange.entrySet()) {
Tablet tablet = null;
boolean successful = false;
-
+
final KeyExtent extentToOpen = entry.getKey();
Text locationToOpen = entry.getValue();
-
+
if (onlineTablets.containsKey(extentToOpen)) {
// know this was from fixing a split, because initial check
// would have caught original extent
@@ -2422,10 +2435,10 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
continue;
}
-
+
try {
TabletResourceManager trm = resourceManager.createTabletResourceManager();
-
+
// this opens the tablet file and fills in the endKey in the
// extent
tablet = new Tablet(TabletServer.this, locationToOpen, extentToOpen, trm, tabletsKeyValues);
@@ -2442,7 +2455,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
if (tablet.getNumEntriesInMemory() > 0 && !tablet.minorCompactNow()) {
throw new RuntimeException("Minor compaction after recovery fails for " + extentToOpen);
}
-
+
Assignment assignment = new Assignment(extentToOpen, getTabletSession());
TabletStateStore.setLocation(assignment);
@@ -2463,7 +2476,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
String table = extent.getTableId().toString();
ProblemReports.getInstance().report(new ProblemReport(table, TABLET_LOAD, extentToOpen.getUUID().toString(), getClientAddressString(), e));
}
-
+
if (!successful) {
synchronized (unopenedTablets) {
synchronized (openingTablets) {
@@ -2498,44 +2511,44 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
}
}
-
+
private FileSystem fs;
private Configuration conf;
private ZooCache cache;
-
+
private SortedMap<KeyExtent,Tablet> onlineTablets = Collections.synchronizedSortedMap(new TreeMap<KeyExtent,Tablet>());
private SortedSet<KeyExtent> unopenedTablets = Collections.synchronizedSortedSet(new TreeSet<KeyExtent>());
private SortedSet<KeyExtent> openingTablets = Collections.synchronizedSortedSet(new TreeSet<KeyExtent>());
@SuppressWarnings("unchecked")
- private Map<KeyExtent,Long> recentlyUnloadedCache = (Map<KeyExtent, Long>)Collections.synchronizedMap(new LRUMap(1000));
-
+ private Map<KeyExtent,Long> recentlyUnloadedCache = Collections.synchronizedMap(new LRUMap(1000));
+
private Thread majorCompactorThread;
-
+
// used for stopping the server and MasterListener thread
private volatile boolean serverStopRequested = false;
-
+
private InetSocketAddress clientAddress;
-
+
private TabletServerResourceManager resourceManager;
private Authenticator authenticator;
private volatile boolean majorCompactorDisabled = false;
-
+
private volatile boolean shutdownComplete = false;
-
+
private ZooLock tabletServerLock;
-
+
private TServer server;
-
+
private DistributedWorkQueue bulkFailedCopyQ;
-
+
private static final String METRICS_PREFIX = "tserver";
-
+
private static ObjectName OBJECT_NAME = null;
-
+
public TabletStatsKeeper getStatsKeeper() {
return statsKeeper;
}
-
+
public Set<String> getLoggers() throws TException, MasterNotRunningException, ThriftSecurityException {
Set<String> allLoggers = new HashSet<String>();
String dir = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZLOGGERS;
@@ -2556,7 +2569,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
log.warn("strategy returned no useful loggers");
return result;
}
-
+
public void addLoggersToMetadata(List<RemoteLogger> logs, KeyExtent extent, int id) {
log.info("Adding " + logs.size() + " logs for extent " + extent + " as alias " + id);
if (!this.onlineTablets.containsKey(extent)) {
@@ -2564,7 +2577,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
// if it doesn't, we'll need to do the same recovery with the old files.
return;
}
-
+
List<MetadataTable.LogEntry> entries = new ArrayList<MetadataTable.LogEntry>();
long now = RelativeTime.currentTimeMillis();
List<String> logSet = new ArrayList<String>();
@@ -2582,14 +2595,14 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
MetadataTable.addLogEntries(SecurityConstants.getSystemCredentials(), entries, getLock());
}
-
+
private int startServer(Property portHint, TProcessor processor, String threadName) throws UnknownHostException {
ServerPort sp = TServerUtils.startServer(portHint, processor, this.getClass().getSimpleName(), threadName, Property.TSERV_PORTSEARCH,
Property.TSERV_MINTHREADS, Property.TSERV_THREADCHECK);
this.server = sp.server;
return sp.port;
}
-
+
private String getMasterAddress() {
try {
List<String> locations = HdfsZooInstance.getInstance().getMasterLocations();
@@ -2599,10 +2612,10 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
} catch (Exception e) {
log.warn("Failed to obtain master host " + e);
}
-
+
return null;
}
-
+
// Connect to the master for posting asynchronous results
private MasterClientService.Iface masterConnection(String address) {
try {
@@ -2618,11 +2631,11 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
return null;
}
-
+
private void returnMasterConnection(MasterClientService.Iface client) {
ThriftUtil.returnClient(client);
}
-
+
private int startTabletClientService() throws UnknownHostException {
// start listening for client connection last
TabletClientService.Iface tch = TraceWrap.service(new ThriftClientHandler());
@@ -2631,25 +2644,26 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
log.info("port = " + port);
return port;
}
-
+
ZooLock getLock() {
return tabletServerLock;
}
-
+
private void announceExistence() {
IZooReaderWriter zoo = ZooReaderWriter.getInstance();
try {
String zPath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZTSERVERS + "/" + getClientAddressString();
-
+
zoo.putPersistentData(zPath, new byte[] {}, NodeExistsPolicy.SKIP);
-
+
tabletServerLock = new ZooLock(zPath);
-
+
LockWatcher lw = new LockWatcher() {
-
+
@Override
public void lostLock(final LockLossReason reason) {
Halt.halt(0, new Runnable() {
+ @Override
public void run() {
if (!serverStopRequested)
log.fatal("Lost tablet server lock (reason = " + reason + "), exiting.");
@@ -2657,7 +2671,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
});
}
-
+
@Override
public void unableToMonitorLockNode(final Throwable e) {
Halt.halt(0, new Runnable() {
@@ -2666,14 +2680,14 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
log.fatal("Lost ability to monitor tablet server lock, exiting.", e);
}
});
-
+
}
};
-
+
byte[] lockContent = new ServerServices(getClientAddressString(), Service.TSERV_CLIENT).toString().getBytes();
for (int i = 0; i < 120 / 5; i++) {
zoo.putPersistentData(zPath, new byte[0], NodeExistsPolicy.SKIP);
-
+
if (tabletServerLock.tryLock(lw, lockContent)) {
log.debug("Obtained tablet server lock " + tabletServerLock.getLockPath());
return;
@@ -2689,7 +2703,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
throw new RuntimeException(e);
}
}
-
+
// main loop listens for client requests
public void run() {
SecurityUtil.serverLogin();
@@ -2706,10 +2720,9 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
clientAddress = new InetSocketAddress(clientAddress.getAddress(), clientPort);
announceExistence();
-
+
ThreadPoolExecutor distWorkQThreadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(
- ServerConfiguration.getSystemConfiguration().getCount(Property.TSERV_WORKQ_THREADS),
- new NamingThreadFactory("distributed work queue"));
+ ServerConfiguration.getSystemConfiguration().getCount(Property.TSERV_WORKQ_THREADS), new NamingThreadFactory("distributed work queue"));
bulkFailedCopyQ = new DistributedWorkQueue(Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZBULK_FAILED_COPYQ);
try {
@@ -2727,27 +2740,27 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
} catch (Exception e) {
log.error("Error registering with JMX", e);
}
-
+
String masterHost;
while (!serverStopRequested) {
// send all of the pending messages
try {
MasterMessage mm = null;
MasterClientService.Iface iface = null;
-
+
try {
// wait until a message is ready to send, or a sever stop
// was requested
while (mm == null && !serverStopRequested) {
mm = masterMessages.poll(1000, TimeUnit.MILLISECONDS);
}
-
+
// have a message to send to the master, so grab a
// connection
masterHost = getMasterAddress();
iface = masterConnection(masterHost);
TServiceClient client = (TServiceClient) iface;
-
+
// if while loop does not execute at all and mm != null,
// then
// finally block should place mm back on queue
@@ -2762,25 +2775,25 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
mm = null;
throw ex;
}
-
+
// if any messages are immediately available grab em and
// send them
mm = masterMessages.poll();
}
-
+
} finally {
-
+
if (mm != null) {
masterMessages.putFirst(mm);
}
returnMasterConnection(iface);
-
+
UtilWaitThread.sleep(1000);
}
} catch (InterruptedException e) {
log.info("Interrupt Exception received, shutting down");
serverStopRequested = true;
-
+
} catch (Exception e) {
// may have lost connection with master
// loop back to the beginning and wait for a new one
@@ -2788,7 +2801,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
log.error(getClientAddressString() + ": TServerInfo: Exception. Master down?", e);
}
}
-
+
// wait for shutdown
// if the main thread exits oldServer the master listener, the JVM will
// kill the
@@ -2808,27 +2821,27 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
log.debug("Stopping Thrift Servers");
TServerUtils.stopTServer(server);
-
+
try {
log.debug("Closing filesystem");
fs.close();
} catch (IOException e) {
log.warn("Failed to close filesystem : " + e.getMessage(), e);
}
-
+
logGCInfo();
-
+
log.info("TServerInfo: stop requested. exiting ... ");
-
+
try {
tabletServerLock.unlock();
} catch (Exception e) {
log.warn("Failed to release tablet server lock", e);
}
}
-
+
private long totalMinorCompactions;
-
+
public static SortedMap<KeyExtent,Text> verifyTabletInformation(KeyExtent extent, TServerInstance instance, SortedMap<Key,Value> tabletsKeyValues,
String clientAddress, ZooLock lock) throws AccumuloSecurityException, DistributedStoreException {
for (int tries = 0; tries < 3; tries++) {
@@ -2849,29 +2862,29 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
set.put(extent, new Text(Constants.ZROOT_TABLET));
return set;
}
-
+
List<ColumnFQ> columnsToFetch = Arrays.asList(new ColumnFQ[] {Constants.METADATA_DIRECTORY_COLUMN, Constants.METADATA_PREV_ROW_COLUMN,
Constants.METADATA_SPLIT_RATIO_COLUMN, Constants.METADATA_OLD_PREV_ROW_COLUMN, Constants.METADATA_TIME_COLUMN});
-
+
if (tabletsKeyValues == null) {
tabletsKeyValues = new TreeMap<Key,Value>();
}
MetadataTable.getTabletAndPrevTabletKeyValues(tabletsKeyValues, extent, null, SecurityConstants.getSystemCredentials());
-
+
SortedMap<Text,SortedMap<ColumnFQ,Value>> tabletEntries;
tabletEntries = MetadataTable.getTabletEntries(tabletsKeyValues, columnsToFetch);
-
+
if (tabletEntries.size() == 0) {
log.warn("Failed to find any metadata entries for " + extent);
return null;
}
-
+
// ensure lst key in map is same as extent that was passed in
if (!tabletEntries.lastKey().equals(extent.getMetadataEntry())) {
log.warn("Failed to find metadata entry for " + extent + " found " + tabletEntries.lastKey());
return null;
}
-
+
TServerInstance future = null;
Text metadataEntry = extent.getMetadataEntry();
for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
@@ -2894,18 +2907,18 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
log.warn("Table " + extent + " has been assigned to " + future + " which is not " + instance);
return null;
}
-
+
// look for incomplete splits
int splitsFixed = 0;
for (Entry<Text,SortedMap<ColumnFQ,Value>> entry : tabletEntries.entrySet()) {
-
+
if (extent.getPrevEndRow() != null) {
Text prevRowMetadataEntry = new Text(KeyExtent.getMetadataEntry(extent.getTableId(), extent.getPrevEndRow()));
if (entry.getKey().compareTo(prevRowMetadataEntry) <= 0) {
continue;
}
}
-
+
if (entry.getValue().containsKey(Constants.METADATA_OLD_PREV_ROW_COLUMN)) {
KeyExtent fixedke = MetadataTable.fixSplit(entry.getKey(), entry.getValue(), instance, SecurityConstants.getSystemCredentials(), lock);
if (fixedke != null) {
@@ -2917,48 +2930,48 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
}
}
-
+
if (splitsFixed > 0) {
// reread and reverify metadata entries now that metadata
// entries were fixed
tabletsKeyValues.clear();
return verifyTabletInformation(extent, instance, null, clientAddress, lock);
}
-
+
SortedMap<KeyExtent,Text> children = new TreeMap<KeyExtent,Text>();
-
+
for (Entry<Text,SortedMap<ColumnFQ,Value>> entry : tabletEntries.entrySet()) {
if (extent.getPrevEndRow() != null) {
Text prevRowMetadataEntry = new Text(KeyExtent.getMetadataEntry(extent.getTableId(), extent.getPrevEndRow()));
-
+
if (entry.getKey().compareTo(prevRowMetadataEntry) <= 0) {
continue;
}
}
-
+
Value prevEndRowIBW = entry.getValue().get(Constants.METADATA_PREV_ROW_COLUMN);
if (prevEndRowIBW == null) {
log.warn("Metadata entry does not have prev row (" + entry.getKey() + ")");
return null;
}
-
+
Value dirIBW = entry.getValue().get(Constants.METADATA_DIRECTORY_COLUMN);
if (dirIBW == null) {
log.warn("Metadata entry does not have directory (" + entry.getKey() + ")");
return null;
}
-
+
Text dir = new Text(dirIBW.get());
-
+
KeyExtent child = new KeyExtent(entry.getKey(), prevEndRowIBW);
children.put(child, dir);
}
-
+
if (!MetadataTable.isContiguousRange(extent, new TreeSet<KeyExtent>(children.keySet()))) {
<TRUNCATED>