You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/01/24 03:15:53 UTC
[20/21] git commit: Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Conflicts:
core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/27be9fb0
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/27be9fb0
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/27be9fb0
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 27be9fb07c837b47d7055196204e9c0b6687daa7
Parents: e36ac37 23bb432
Author: Josh Elser <el...@apache.org>
Authored: Thu Jan 23 21:07:21 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Jan 23 21:07:21 2014 -0500
----------------------------------------------------------------------
.../core/client/impl/OfflineScanner.java | 15 +-
.../core/client/mapreduce/RangeInputSplit.java | 11 +-
.../mapreduce/lib/util/ConfiguratorBase.java | 11 +-
.../core/client/mapreduce/InputFormatBase.java | 1634 ------------------
.../test/continuous/ContinuousVerify.java | 21 -
test/system/continuous/run-verify.sh | 24 +-
6 files changed, 14 insertions(+), 1702 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/27be9fb0/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
index 0cb0409,5f3069a..eaf1b76
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
@@@ -117,8 -118,15 +120,9 @@@ class OfflineIterator implements Iterat
private Instance instance;
private ScannerOptions options;
private ArrayList<SortedKeyValueIterator<Key,Value>> readers;
-
+ private AccumuloConfiguration config;
+
- /**
- * @param instance
- * @param credentials
- * @param authorizations
- * @param table
- */
- public OfflineIterator(ScannerOptions options, Instance instance, TCredentials credentials, Authorizations authorizations, Text table, Range range) {
+ public OfflineIterator(ScannerOptions options, Instance instance, Credentials credentials, Authorizations authorizations, Text table, Range range) {
this.options = new ScannerOptions(options);
this.instance = instance;
this.range = range;
@@@ -132,7 -140,8 +136,8 @@@
this.readers = new ArrayList<SortedKeyValueIterator<Key,Value>>();
try {
- conn = instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials));
+ conn = instance.getConnector(credentials.getPrincipal(), credentials.getToken());
+ config = new ConfigurationCopy(conn.instanceOperations().getSiteConfiguration());
nextTablet();
while (iter != null && !iter.hasTop())
@@@ -225,21 -234,14 +230,21 @@@
if (currentExtent != null && !extent.isPreviousExtent(currentExtent))
throw new AccumuloException(" " + currentExtent + " is not previous extent " + extent);
-
- String tablesDir = ServerConfigurationUtil.getConfiguration(instance).get(Property.INSTANCE_DFS_DIR) + "/tables";
+
- String tablesDir = Constants.getTablesDir(config);
++ String tablesDir = config.get(Property.INSTANCE_DFS_DIR) + "/tables";
+
List<String> absFiles = new ArrayList<String>();
for (String relPath : relFiles) {
- if (relPath.startsWith(".."))
- absFiles.add(tablesDir + relPath.substring(2));
- else
- absFiles.add(tablesDir + "/" + tableId + relPath);
+ if (relPath.contains(":")) {
+ absFiles.add(relPath);
+ } else {
+ // handle old-style relative paths
+ if (relPath.startsWith("..")) {
+ absFiles.add(tablesDir + relPath.substring(2));
+ } else {
+ absFiles.add(tablesDir + "/" + tableId + relPath);
+ }
+ }
}
iter = createIterator(extent, absFiles);
@@@ -294,7 -296,9 +299,7 @@@
AccumuloConfiguration acuTableConf = AccumuloConfiguration.getTableConfiguration(conn, tableId);
Configuration conf = CachedConfiguration.getInstance();
-
+
- FileSystem fs = FileUtil.getFileSystem(conf, config);
-
for (SortedKeyValueIterator<Key,Value> reader : readers) {
((FileSKVIterator) reader).close();
}
@@@ -303,7 -307,6 +308,7 @@@
// TODO need to close files - ACCUMULO-1303
for (String file : absFiles) {
- FileSystem fs = FileUtil.getFileSystem(file, conf, ServerConfigurationUtil.getConfiguration(instance));
++ FileSystem fs = FileUtil.getFileSystem(file, conf, config);
FileSKVIterator reader = FileOperations.getInstance().openReader(file, false, fs, conf, acuTableConf, null, null);
readers.add(reader);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/27be9fb0/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
index 831d10b,75f140b..b238903
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
@@@ -36,9 -33,6 +36,7 @@@ import org.apache.accumulo.core.client.
import org.apache.accumulo.core.client.mapreduce.lib.util.InputConfigurator;
import org.apache.accumulo.core.client.mock.MockInstance;
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer;
- import org.apache.accumulo.core.conf.AccumuloConfiguration;
- import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.PartialKey;
@@@ -369,15 -322,8 +367,8 @@@ public class RangeInputSplit extends In
if (null == zooKeepers) {
return null;
}
-
- return new ZooKeeperInstance(getInstanceName(), getZooKeepers());
+
- ZooKeeperInstance zki = new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(getInstanceName()).withZkHosts(getZooKeepers()));
-
- // Wrap the DefaultConfiguration with a SiteConfiguration so we use accumulo-site.xml
- // when it's present
- AccumuloConfiguration xmlConfig = SiteConfiguration.getInstance(zki.getConfiguration());
- zki.setConfiguration(xmlConfig);
-
- return zki;
++ return new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(getInstanceName()).withZkHosts(getZooKeepers()));
}
public String getInstanceName() {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/27be9fb0/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
index b846356,b1ae3a5..8bce08b
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
@@@ -348,21 -232,8 +348,14 @@@ public class ConfiguratorBase
if ("MockInstance".equals(instanceType))
return new MockInstance(conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME)));
else if ("ZooKeeperInstance".equals(instanceType)) {
- ZooKeeperInstance zki;
- return new ZooKeeperInstance(conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME)), conf.get(enumToConfKey(implementingClass,
- InstanceOpts.ZOO_KEEPERS)));
+ String clientConfigString = conf.get(enumToConfKey(implementingClass, InstanceOpts.CLIENT_CONFIG));
+ if (clientConfigString == null) {
+ String instanceName = conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME));
+ String zookeepers = conf.get(enumToConfKey(implementingClass, InstanceOpts.ZOO_KEEPERS));
- zki = new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(instanceName).withZkHosts(zookeepers));
++ return new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(instanceName).withZkHosts(zookeepers));
+ } else {
- zki = new ZooKeeperInstance(ClientConfiguration.deserialize(clientConfigString));
++ return new ZooKeeperInstance(ClientConfiguration.deserialize(clientConfigString));
+ }
-
- // Wrap the DefaultConfiguration with a SiteConfiguration
- AccumuloConfiguration xmlConfig = SiteConfiguration.getInstance(zki.getConfiguration());
- zki.setConfiguration(xmlConfig);
-
- return zki;
} else if (instanceType.isEmpty())
throw new IllegalStateException("Instance has not been configured for " + implementingClass.getSimpleName());
else
http://git-wip-us.apache.org/repos/asf/accumulo/blob/27be9fb0/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/27be9fb0/test/system/continuous/run-verify.sh
----------------------------------------------------------------------