You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/01/31 05:02:00 UTC
[08/10] git commit: Fix an InputFormatBase blunder,
add defensive copies, and lots of UTF8
Fix an InputFormatBase blunder, add defensive copies, and lots of UTF8
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0df56927
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0df56927
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0df56927
Branch: refs/heads/2292-findbugs
Commit: 0df5692789da48996c47d3f960a1c05bf40e533d
Parents: 6770ed9
Author: Josh Elser <el...@apache.org>
Authored: Thu Jan 30 21:45:30 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Jan 30 21:45:30 2014 -0500
----------------------------------------------------------------------
.../accumulo/core/client/mapred/InputFormatBase.java | 2 +-
.../core/client/mapreduce/InputFormatBase.java | 14 +++++++-------
.../core/client/mapreduce/RangeInputSplit.java | 6 +++---
.../mapreduce/lib/partition/RangePartitioner.java | 7 +++++--
.../client/mapreduce/lib/util/InputConfigurator.java | 12 ++++++------
.../apache/accumulo/core/client/mock/MockShell.java | 5 +++--
.../core/client/security/tokens/PasswordToken.java | 2 +-
.../accumulo/core/conf/DefaultConfiguration.java | 8 +++++---
8 files changed, 31 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0df56927/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java b/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
index 8cc4d18..ae361de 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
@@ -619,7 +619,7 @@ public abstract class InputFormatBase<K,V> implements InputFormat<K,V> {
Connector conn = instance.getConnector(principal, token);
log.debug("Creating scanner for table: " + table);
log.debug("Authorizations are: " + authorizations);
- if (isOfflineScan(job)) {
+ if (isOffline) {
String tokenClass = token.getClass().getCanonicalName();
ByteBuffer tokenBuffer = ByteBuffer.wrap(CredentialHelper.toBytes(token));
scanner = new OfflineScanner(instance, new TCredentials(principal, tokenClass, tokenBuffer, instance.getInstanceID()), Tables.getTableId(instance,
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0df56927/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
index 4c57d1d..4516176 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
@@ -631,7 +631,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
Connector conn = instance.getConnector(principal, token);
log.debug("Creating scanner for table: " + table);
log.debug("Authorizations are: " + authorizations);
- if (isOfflineScan(attempt)) {
+ if (isOffline) {
String tokenClass = token.getClass().getCanonicalName();
ByteBuffer tokenBuffer = ByteBuffer.wrap(CredentialHelper.toBytes(token));
scanner = new OfflineScanner(instance, new TCredentials(principal, tokenClass, tokenBuffer, instance.getInstanceID()), Tables.getTableId(instance,
@@ -1180,8 +1180,8 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
List<IteratorSetting> iteratorSettings = InputConfigurator.getIterators(CLASS, conf);
List<AccumuloIterator> deprecatedIterators = new ArrayList<AccumuloIterator>(iteratorSettings.size());
for (IteratorSetting setting : iteratorSettings) {
- AccumuloIterator deprecatedIter = new AccumuloIterator(new String(setting.getPriority() + AccumuloIterator.FIELD_SEP + setting.getIteratorClass()
- + AccumuloIterator.FIELD_SEP + setting.getName()));
+ AccumuloIterator deprecatedIter = new AccumuloIterator(setting.getPriority() + AccumuloIterator.FIELD_SEP + setting.getIteratorClass()
+ + AccumuloIterator.FIELD_SEP + setting.getName());
deprecatedIterators.add(deprecatedIter);
}
return deprecatedIterators;
@@ -1198,8 +1198,8 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
for (Entry<String,String> opt : setting.getOptions().entrySet()) {
String deprecatedOption;
try {
- deprecatedOption = new String(setting.getName() + AccumuloIteratorOption.FIELD_SEP + URLEncoder.encode(opt.getKey(), "UTF-8")
- + AccumuloIteratorOption.FIELD_SEP + URLEncoder.encode(opt.getValue(), "UTF-8"));
+ deprecatedOption = setting.getName() + AccumuloIteratorOption.FIELD_SEP + URLEncoder.encode(opt.getKey(), "UTF-8")
+ + AccumuloIteratorOption.FIELD_SEP + URLEncoder.encode(opt.getValue(), "UTF-8");
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
@@ -1250,7 +1250,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
@Override
public String toString() {
- return new String(priority + FIELD_SEP + iteratorClass + FIELD_SEP + iteratorName);
+ return priority + FIELD_SEP + iteratorClass + FIELD_SEP + iteratorName;
}
}
@@ -1299,7 +1299,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
@Override
public String toString() {
try {
- return new String(iteratorName + FIELD_SEP + URLEncoder.encode(key, "UTF-8") + FIELD_SEP + URLEncoder.encode(value, "UTF-8"));
+ return iteratorName + FIELD_SEP + URLEncoder.encode(key, "UTF-8") + FIELD_SEP + URLEncoder.encode(value, "UTF-8");
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0df56927/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
index 75f140b..73c9b59 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
@@ -67,7 +67,7 @@ public class RangeInputSplit extends InputSplit implements Writable {
public RangeInputSplit(Range range, String[] locations) {
this.range = range;
- this.locations = locations;
+ setLocations(locations);
}
public Range getRange() {
@@ -138,7 +138,7 @@ public class RangeInputSplit extends InputSplit implements Writable {
@Override
public String[] getLocations() throws IOException {
- return locations;
+ return Arrays.copyOf(locations, locations.length);
}
@Override
@@ -368,7 +368,7 @@ public class RangeInputSplit extends InputSplit implements Writable {
}
public void setLocations(String[] locations) {
- this.locations = locations;
+ this.locations = Arrays.copyOf(locations, locations.length);
}
public Boolean isMockInstance() {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0df56927/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
index 920bd5b..e29c791 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
@@ -17,14 +17,17 @@
package org.apache.accumulo.core.client.mapreduce.lib.partition;
import java.io.BufferedReader;
+import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
+import java.io.InputStreamReader;
import java.net.URI;
import java.util.Arrays;
import java.util.Scanner;
import java.util.TreeSet;
+import org.apache.accumulo.core.Constants;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
@@ -87,10 +90,10 @@ public class RangePartitioner extends Partitioner<Text,Writable> implements Conf
for (Path path : cf) {
if (path.toUri().getPath().endsWith(cutFileName.substring(cutFileName.lastIndexOf('/')))) {
TreeSet<Text> cutPoints = new TreeSet<Text>();
- Scanner in = new Scanner(new BufferedReader(new FileReader(path.toString())));
+ Scanner in = new Scanner(new BufferedReader(new InputStreamReader(new FileInputStream(path.toString()), Constants.UTF8)));
try {
while (in.hasNextLine())
- cutPoints.add(new Text(Base64.decodeBase64(in.nextLine().getBytes())));
+ cutPoints.add(new Text(Base64.decodeBase64(in.nextLine().getBytes(Constants.UTF8))));
} finally {
in.close();
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0df56927/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java
index b0e649b..d9e313c 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java
@@ -138,7 +138,7 @@ public class InputConfigurator extends ConfiguratorBase {
*/
public static Authorizations getScanAuthorizations(Class<?> implementingClass, Configuration conf) {
String authString = conf.get(enumToConfKey(implementingClass, ScanOpts.AUTHORIZATIONS));
- return authString == null ? Constants.NO_AUTHS : new Authorizations(authString.getBytes());
+ return authString == null ? Constants.NO_AUTHS : new Authorizations(authString.getBytes(Constants.UTF8));
}
/**
@@ -216,9 +216,9 @@ public class InputConfigurator extends ConfiguratorBase {
if (column.getFirst() == null)
throw new IllegalArgumentException("Column family can not be null");
- String col = new String(Base64.encodeBase64(TextUtil.getBytes(column.getFirst())));
+ String col = new String(Base64.encodeBase64(TextUtil.getBytes(column.getFirst())), Constants.UTF8);
if (column.getSecond() != null)
- col += ":" + new String(Base64.encodeBase64(TextUtil.getBytes(column.getSecond())));
+ col += ":" + new String(Base64.encodeBase64(TextUtil.getBytes(column.getSecond())), Constants.UTF8);
columnStrings.add(col);
}
@@ -258,8 +258,8 @@ public class InputConfigurator extends ConfiguratorBase {
for (String col : serialized) {
int idx = col.indexOf(":");
- Text cf = new Text(idx < 0 ? Base64.decodeBase64(col.getBytes(Charset.forName("UTF-8"))) : Base64.decodeBase64(col.substring(0, idx).getBytes(Charset.forName("UTF-8"))));
- Text cq = idx < 0 ? null : new Text(Base64.decodeBase64(col.substring(idx + 1).getBytes()));
+ Text cf = new Text(idx < 0 ? Base64.decodeBase64(col.getBytes(Constants.UTF8)) : Base64.decodeBase64(col.substring(0, idx).getBytes(Constants.UTF8)));
+ Text cq = idx < 0 ? null : new Text(Base64.decodeBase64(col.substring(idx + 1).getBytes(Constants.UTF8)));
columns.add(new Pair<Text,Text>(cf, cq));
}
return columns;
@@ -323,7 +323,7 @@ public class InputConfigurator extends ConfiguratorBase {
try {
while (tokens.hasMoreTokens()) {
String itstring = tokens.nextToken();
- ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(itstring.getBytes()));
+ ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(itstring.getBytes(Constants.UTF8)));
list.add(new IteratorSetting(new DataInputStream(bais)));
bais.close();
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0df56927/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java b/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
index 4469d5c..c97e2b4 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
@@ -24,6 +24,7 @@ import java.io.Writer;
import jline.ConsoleReader;
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.util.shell.Shell;
import org.apache.commons.cli.CommandLine;
@@ -77,7 +78,7 @@ public class MockShell extends Shell {
printInfo();
if (execFile != null) {
- java.util.Scanner scanner = new java.util.Scanner(new File(execFile));
+ java.util.Scanner scanner = new java.util.Scanner(new File(execFile), Constants.UTF8.name());
try {
while (scanner.hasNextLine() && !hasExited()) {
execCommand(scanner.nextLine(), true, isVerbose());
@@ -137,6 +138,6 @@ public class MockShell extends Shell {
sb.append(command).append(NEWLINE);
}
- return new ByteArrayInputStream(sb.toString().getBytes());
+ return new ByteArrayInputStream(sb.toString().getBytes(Constants.UTF8));
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0df56927/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java b/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
index 50d6938..9386b56 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
@@ -40,7 +40,7 @@ public class PasswordToken implements AuthenticationToken {
private byte[] password = null;
public byte[] getPassword() {
- return password;
+ return Arrays.copyOf(password, password.length);
}
/**
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0df56927/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java b/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
index 58f446d..d653274 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
@@ -20,11 +20,13 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.TreeMap;
+import org.apache.accumulo.core.Constants;
import org.apache.log4j.Logger;
public class DefaultConfiguration extends AccumuloConfiguration {
@@ -62,7 +64,7 @@ public class DefaultConfiguration extends AccumuloConfiguration {
int n;
try {
while ((n = data.read(buffer)) > 0)
- doc.print(new String(buffer, 0, n));
+ doc.print(new String(buffer, 0, n, Constants.UTF8));
} catch (IOException e) {
e.printStackTrace();
return;
@@ -176,9 +178,9 @@ public class DefaultConfiguration extends AccumuloConfiguration {
/*
* Generate documentation for conf/accumulo-site.xml file usage
*/
- public static void main(String[] args) throws FileNotFoundException {
+ public static void main(String[] args) throws FileNotFoundException, UnsupportedEncodingException {
if (args.length == 2 && args[0].equals("--generate-doc")) {
- generateDocumentation(new PrintStream(args[1]));
+ generateDocumentation(new PrintStream(args[1], Constants.UTF8.name()));
} else {
throw new IllegalArgumentException("Usage: " + DefaultConfiguration.class.getName() + " --generate-doc <filename>");
}