You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2015/02/06 00:00:01 UTC
accumulo git commit: ACCUMULO-2462 Remove confusing indirect use of
instance.dfs.uri
Repository: accumulo
Updated Branches:
refs/heads/master 8c2294df8 -> d7bb9ac17
ACCUMULO-2462 Remove confusing indirect use of instance.dfs.uri
This removes a confusing indirect usage of the deprecated
instance.dfs.uri property, to resolve relative paths on the command-line
of PrintInfo. This removes the reliance on accumulo-site.xml for
determining which filesystem to use when not specifying a
fully-qualified file name.
There remains some reliance on accumulo-site.xml, however, due to
ACCUMULO-3566. It also doesn't remove the somewhat confusing behavior of
automatically looking at the local filesystem if the default hadoop
filesystem doesn't contain the file specified. To handle that special
case, I print out a message showing the actual file which it resolves
to.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d7bb9ac1
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d7bb9ac1
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d7bb9ac1
Branch: refs/heads/master
Commit: d7bb9ac1748a8d794282a99045c7104bd7e9513a
Parents: 8c2294d
Author: Christopher Tubbs <ct...@apache.org>
Authored: Thu Feb 5 17:31:07 2015 -0500
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Thu Feb 5 17:31:07 2015 -0500
----------------------------------------------------------------------
.../accumulo/core/file/rfile/PrintInfo.java | 19 ++++++-------------
.../core/file/rfile/bcfile/PrintInfo.java | 12 ++++--------
2 files changed, 10 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d7bb9ac1/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
index 591d477..37b3962 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
@@ -22,7 +22,6 @@ import java.util.Map;
import java.util.Map.Entry;
import org.apache.accumulo.core.cli.Help;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.data.ByteSequence;
@@ -31,7 +30,6 @@ import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile;
import org.apache.accumulo.core.file.rfile.RFile.Reader;
-import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.start.spi.KeywordExecutable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -72,13 +70,7 @@ public class PrintInfo implements KeywordExecutable {
public void execute(final String[] args) throws Exception {
Configuration conf = new Configuration();
- AccumuloConfiguration aconf = SiteConfiguration.getInstance(DefaultConfiguration.getInstance());
- // TODO ACCUMULO-2462 This will only work for RFiles (path only, not URI) in HDFS when the correct filesystem for the given file
- // is on Property.INSTANCE_DFS_DIR or, when INSTANCE_DFS_DIR is not defined, is on the default filesystem
- // defined in the Hadoop's core-site.xml
- //
- // A workaround is to always provide a URI to this class
- FileSystem hadoopFs = VolumeConfiguration.getDefaultVolume(conf, aconf).getFileSystem();
+ FileSystem hadoopFs = FileSystem.get(conf);
FileSystem localFs = FileSystem.getLocal(conf);
Opts opts = new Opts();
opts.parseArgs(PrintInfo.class.getName(), args);
@@ -97,14 +89,15 @@ public class PrintInfo implements KeywordExecutable {
if (arg.contains(":"))
fs = path.getFileSystem(conf);
else {
- // Recommend a URI is given for the above todo reason
log.warn("Attempting to find file across filesystems. Consider providing URI instead of path");
fs = hadoopFs.exists(path) ? hadoopFs : localFs; // fall back to local
}
+ System.out.println("Working on: " + path.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString());
- CachableBlockFile.Reader _rdr = new CachableBlockFile.Reader(fs, path, conf, null, null, aconf);
+ CachableBlockFile.Reader _rdr = new CachableBlockFile.Reader(fs, path, conf, null, null,
+ SiteConfiguration.getInstance(DefaultConfiguration.getInstance()));
Reader iter = new RFile.Reader(_rdr);
- MetricsGatherer<Map<String, ArrayList<VisibilityMetric>>> vmg = new VisMetricsGatherer();
+ MetricsGatherer<Map<String,ArrayList<VisibilityMetric>>> vmg = new VisMetricsGatherer();
if (opts.vis || opts.hash)
iter.registerMetrics(vmg);
@@ -113,7 +106,7 @@ public class PrintInfo implements KeywordExecutable {
System.out.println();
org.apache.accumulo.core.file.rfile.bcfile.PrintInfo.main(new String[] {arg});
- Map<String, ArrayList<ByteSequence>> localityGroupCF = null;
+ Map<String,ArrayList<ByteSequence>> localityGroupCF = null;
if (opts.histogram || opts.dump || opts.vis || opts.hash) {
localityGroupCF = iter.getLocalityGroupCF();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d7bb9ac1/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
index 9ac1e96..7b2abe8 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
@@ -21,22 +21,20 @@ import java.io.PrintStream;
import java.util.Map.Entry;
import java.util.Set;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.file.rfile.bcfile.BCFile.MetaIndexEntry;
-import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class PrintInfo {
- public static void printMetaBlockInfo(Configuration conf, FileSystem fs, Path path, AccumuloConfiguration accumuloConfiguration) throws IOException {
+ public static void printMetaBlockInfo(Configuration conf, FileSystem fs, Path path) throws IOException {
FSDataInputStream fsin = fs.open(path);
BCFile.Reader bcfr = null;
try {
- bcfr = new BCFile.Reader(fsin, fs.getFileStatus(path).getLen(), conf, accumuloConfiguration);
+ bcfr = new BCFile.Reader(fsin, fs.getFileStatus(path).getLen(), conf, SiteConfiguration.getInstance(DefaultConfiguration.getInstance()));
Set<Entry<String,MetaIndexEntry>> es = bcfr.metaIndex.index.entrySet();
@@ -57,9 +55,7 @@ public class PrintInfo {
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
- AccumuloConfiguration siteConf = SiteConfiguration.getInstance(DefaultConfiguration.getInstance());
- // TODO ACCUMULO-2462 not going to operate as expected with volumes when a path, not URI, is given
- FileSystem hadoopFs = VolumeConfiguration.getDefaultVolume(conf, siteConf).getFileSystem();
+ FileSystem hadoopFs = FileSystem.get(conf);
FileSystem localFs = FileSystem.getLocal(conf);
Path path = new Path(args[0]);
FileSystem fs;
@@ -67,6 +63,6 @@ public class PrintInfo {
fs = path.getFileSystem(conf);
else
fs = hadoopFs.exists(path) ? hadoopFs : localFs; // fall back to local
- printMetaBlockInfo(conf, fs, path, siteConf);
+ printMetaBlockInfo(conf, fs, path);
}
}