You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2013/06/18 19:45:02 UTC
svn commit: r1494226 [1/2] - in /accumulo/branches/ACCUMULO-118:
core/src/main/java/org/apache/accumulo/core/conf/
server/src/main/java/org/apache/accumulo/server/
server/src/main/java/org/apache/accumulo/server/fs/
server/src/main/java/org/apache/accu...
Author: ecn
Date: Tue Jun 18 17:45:01 2013
New Revision: 1494226
URL: http://svn.apache.org/r1494226
Log:
ACCUMULO-118 switch to using "volume" instead of "namespace"
Added:
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java (contents, props changed)
- copied, changed from r1494136, accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/FileSystem.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java (contents, props changed)
- copied, changed from r1494136, accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/FileSystemImpl.java
Removed:
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/FileSystem.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/FileSystemImpl.java
Modified:
accumulo/branches/ACCUMULO-118/core/src/main/java/org/apache/accumulo/core/conf/Property.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/Accumulo.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/ServerConstants.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/FileRef.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/Master.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/LogCloser.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/state/SetGoalState.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/CreateTable.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/DeleteTable.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/ExportTable.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/ImportTable.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/monitor/Monitor.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/FileManager.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/MinorCompactor.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/MultiReader.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecovery.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/TabletServerLogger.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/CheckForMetadataProblems.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/FileUtil.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/OfflineMetadataScanner.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java
accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/MultiReaderTest.java
accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java
accumulo/branches/ACCUMULO-118/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousStatsCollector.java
accumulo/branches/ACCUMULO-118/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
Modified: accumulo/branches/ACCUMULO-118/core/src/main/java/org/apache/accumulo/core/conf/Property.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/core/src/main/java/org/apache/accumulo/core/conf/Property.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/core/src/main/java/org/apache/accumulo/core/conf/Property.java (original)
+++ accumulo/branches/ACCUMULO-118/core/src/main/java/org/apache/accumulo/core/conf/Property.java Tue Jun 18 17:45:01 2013
@@ -79,7 +79,7 @@ public enum Property {
"A secret unique to a given instance that all servers must know in order to communicate with one another."
+ " Change it before initialization. To change it later use ./bin/accumulo accumulo.server.util.ChangeSecret [oldpasswd] [newpasswd], "
+ " and then update conf/accumulo-site.xml everywhere."),
- INSTANCE_NAMESPACES("instance.namespaces", "", PropertyType.STRING, "A list of namespaces to use."),
+ INSTANCE_VOLUMES("instance.volumes", "", PropertyType.STRING, "A list of volumes to use. By default, this will be the namenode in the hadoop configuration in the accumulo classpath."),
INSTANCE_SECURITY_AUTHENTICATOR("instance.security.authenticator", "org.apache.accumulo.server.security.handler.ZKAuthenticator", PropertyType.CLASSNAME,
"The authenticator class that accumulo will use to determine if a user has privilege to perform an action"),
INSTANCE_SECURITY_AUTHORIZOR("instance.security.authorizor", "org.apache.accumulo.server.security.handler.ZKAuthorizor", PropertyType.CLASSNAME,
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/Accumulo.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/Accumulo.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/Accumulo.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/Accumulo.java Tue Jun 18 17:45:01 2013
@@ -32,7 +32,7 @@ import org.apache.accumulo.core.util.Uti
import org.apache.accumulo.core.util.Version;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.ServerConfiguration;
-import org.apache.accumulo.server.fs.FileSystem;
+import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.util.time.SimpleTimer;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
import org.apache.hadoop.fs.FileStatus;
@@ -46,7 +46,7 @@ public class Accumulo {
private static final Logger log = Logger.getLogger(Accumulo.class);
- public static synchronized void updateAccumuloVersion(FileSystem fs) {
+ public static synchronized void updateAccumuloVersion(VolumeManager fs) {
try {
if (getAccumuloPersistentVersion(fs) == Constants.PREV_DATA_VERSION) {
fs.create(new Path(ServerConstants.getDataVersionLocation() + "/" + Constants.DATA_VERSION));
@@ -57,10 +57,10 @@ public class Accumulo {
}
}
- public static synchronized int getAccumuloPersistentVersion(FileSystem fs) {
+ public static synchronized int getAccumuloPersistentVersion(VolumeManager fs) {
int dataVersion;
try {
- FileStatus[] files = fs.getDefaultNamespace().listStatus(ServerConstants.getDataVersionLocation());
+ FileStatus[] files = fs.getDefaultVolume().listStatus(ServerConstants.getDataVersionLocation());
if (files == null || files.length == 0) {
dataVersion = -1; // assume it is 0.5 or earlier
} else {
@@ -80,7 +80,7 @@ public class Accumulo {
}
}
- public static void init(FileSystem fs, ServerConfiguration config, String application) throws UnknownHostException {
+ public static void init(VolumeManager fs, ServerConfiguration config, String application) throws UnknownHostException {
System.setProperty("org.apache.accumulo.core.application", application);
@@ -181,7 +181,7 @@ public class Accumulo {
return result.getHostName();
}
- public static void waitForZookeeperAndHdfs(FileSystem fs) {
+ public static void waitForZookeeperAndHdfs(VolumeManager fs) {
log.info("Attempting to talk to zookeeper");
while (true) {
try {
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/ServerConstants.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/ServerConstants.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/ServerConstants.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/ServerConstants.java Tue Jun 18 17:45:01 2013
@@ -30,7 +30,7 @@ public class ServerConstants {
// these are functions to delay loading the Accumulo configuration unless we must
public static String[] getBaseDirs() {
String singleNamespace = ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_DFS_DIR);
- String ns = ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_NAMESPACES);
+ String ns = ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_VOLUMES);
if (ns == null || ns.isEmpty()) {
Configuration hadoopConfig = CachedConfiguration.getInstance();
String fullPath = hadoopConfig.get("fs.default.name") + singleNamespace;
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/FileRef.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/FileRef.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/FileRef.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/FileRef.java Tue Jun 18 17:45:01 2013
@@ -30,7 +30,7 @@ public class FileRef implements Comparab
String metaReference; // something like ../2/d-00000/A00001.rf
Path fullReference; // something like hdfs://nn:9001/accumulo/tables/2/d-00000/A00001.rf
- public FileRef(FileSystem fs, Key key) {
+ public FileRef(VolumeManager fs, Key key) {
metaReference = key.getColumnQualifier().toString();
fullReference = fs.getFullPath(key);
}
Copied: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java (from r1494136, accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/FileSystem.java)
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java?p2=accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java&p1=accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/FileSystem.java&r1=1494136&r2=1494226&rev=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/FileSystem.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java Tue Jun 18 17:45:01 2013
@@ -24,12 +24,13 @@ import org.apache.hadoop.fs.ContentSumma
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
/**
* A wrapper around multiple hadoop FileSystem objects, which are assumed to be different namespaces.
*/
-public interface FileSystem {
+public interface VolumeManager {
// close the underlying FileSystems
void close() throws IOException;
@@ -65,18 +66,18 @@ public interface FileSystem {
FileStatus getFileStatus(Path path) throws IOException;
// find the appropriate FileSystem object given a path
- org.apache.hadoop.fs.FileSystem getFileSystemByPath(Path path);
+ FileSystem getFileSystemByPath(Path path);
- org.apache.hadoop.fs.FileSystem getFileSystemByPath(String path);
+ FileSystem getFileSystemByPath(String path);
- // get a mapping of namespace to FileSystem
- Map<String, ? extends org.apache.hadoop.fs.FileSystem> getFileSystems();
+ // get a mapping of volume to FileSystem
+ Map<String, ? extends FileSystem> getFileSystems();
// return the item in options that is in the same namespace as source
Path matchingFileSystem(Path source, String[] options);
- // create a new path in the same namespace as the sourceDir
- String newPathOnSameNamespace(String sourceDir, String suffix);
+ // create a new path in the same volume as the sourceDir
+ String newPathOnSameVolume(String sourceDir, String suffix);
// forward to the appropriate FileSystem object
FileStatus[] listStatus(Path path) throws IOException;
@@ -99,11 +100,11 @@ public interface FileSystem {
// forward to the appropriate FileSystem object
boolean isFile(Path path) throws IOException;
- // all namespaces are ready to provide service (not in SafeMode, for example)
+ // all volume are ready to provide service (not in SafeMode, for example)
boolean isReady() throws IOException;
// ambiguous references to files go here
- org.apache.hadoop.fs.FileSystem getDefaultNamespace();
+ FileSystem getDefaultVolume();
// forward to the appropriate FileSystem object
FileStatus[] globStatus(Path path) throws IOException;
@@ -115,5 +116,5 @@ public interface FileSystem {
Path getFullPath(String paths[], String fileName) throws IOException;
// forward to the appropriate FileSystem object
- ContentSummary getContentSummary(String dir);
+ ContentSummary getContentSummary(Path dir) throws IOException;
}
Propchange: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
------------------------------------------------------------------------------
svn:eol-style = native
Copied: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java (from r1494136, accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/FileSystemImpl.java)
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java?p2=accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java&p1=accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/FileSystemImpl.java&r1=1494136&r2=1494226&rev=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/FileSystemImpl.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java Tue Jun 18 17:45:01 2013
@@ -51,29 +51,29 @@ import org.apache.hadoop.hdfs.Distribute
import org.apache.hadoop.util.Progressable;
import org.apache.log4j.Logger;
-public class FileSystemImpl implements org.apache.accumulo.server.fs.FileSystem {
+public class VolumeManagerImpl implements VolumeManager {
private static final Logger log = Logger.getLogger(FileSystem.class);
- Map<String, ? extends FileSystem> namespaces;
- String defaultNamespace;
+ Map<String, ? extends FileSystem> volumes;
+ String defaultVolumes;
AccumuloConfiguration conf;
- protected FileSystemImpl(Map<String, ? extends FileSystem> namespaces, String defaultNamespace, AccumuloConfiguration conf) {
- this.namespaces = namespaces;
- this.defaultNamespace = defaultNamespace;
+ protected VolumeManagerImpl(Map<String, ? extends FileSystem> volumes, String defaultVolume, AccumuloConfiguration conf) {
+ this.volumes = volumes;
+ this.defaultVolumes = defaultVolume;
this.conf = conf;
ensureSyncIsEnabled();
}
- public static org.apache.accumulo.server.fs.FileSystem getLocal() throws IOException {
- return new FileSystemImpl(Collections.singletonMap("", FileSystem.getLocal(CachedConfiguration.getInstance())), "", DefaultConfiguration.getDefaultConfiguration());
+ public static org.apache.accumulo.server.fs.VolumeManager getLocal() throws IOException {
+ return new VolumeManagerImpl(Collections.singletonMap("", FileSystem.getLocal(CachedConfiguration.getInstance())), "", DefaultConfiguration.getDefaultConfiguration());
}
@Override
public void close() throws IOException {
IOException ex = null;
- for (FileSystem fs : namespaces.values()) {
+ for (FileSystem fs : volumes.values()) {
try {
fs.close();
} catch (IOException e) {
@@ -250,7 +250,7 @@ public class FileSystemImpl implements o
}
}
- return namespaces.get(defaultNamespace);
+ return volumes.get(defaultVolumes);
}
@Override
@@ -259,8 +259,8 @@ public class FileSystemImpl implements o
}
@Override
- public Map<String, ? extends org.apache.hadoop.fs.FileSystem> getFileSystems() {
- return namespaces;
+ public Map<String, ? extends FileSystem> getFileSystems() {
+ return volumes;
}
@Override
@@ -283,7 +283,7 @@ public class FileSystemImpl implements o
FileSystem source = getFileSystemByPath(path);
FileSystem dest = getFileSystemByPath(newPath);
if (source != dest) {
- throw new NotImplementedException("Cannot rename files across namespaces: " + path + " -> " + newPath);
+ throw new NotImplementedException("Cannot rename files across volumes: " + path + " -> " + newPath);
}
return source.rename(path, newPath);
}
@@ -305,18 +305,18 @@ public class FileSystemImpl implements o
return getFileSystemByPath(path).isFile(path);
}
- public static org.apache.accumulo.server.fs.FileSystem get() throws IOException {
+ public static org.apache.accumulo.server.fs.VolumeManager get() throws IOException {
AccumuloConfiguration conf = ServerConfiguration.getSystemConfiguration(HdfsZooInstance.getInstance());
return get(conf);
}
static private final String DEFAULT = "";
- public static org.apache.accumulo.server.fs.FileSystem get(AccumuloConfiguration conf) throws IOException {
+ public static org.apache.accumulo.server.fs.VolumeManager get(AccumuloConfiguration conf) throws IOException {
Map<String, FileSystem> fileSystems = new HashMap<String, FileSystem>();
Configuration hadoopConf = CachedConfiguration.getInstance();
fileSystems.put(DEFAULT, FileSystem.get(hadoopConf));
- String ns = ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_NAMESPACES);
+ String ns = ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_VOLUMES);
if (ns != null) {
for (String space : ns.split(",")) {
if (space.contains(":")) {
@@ -326,7 +326,7 @@ public class FileSystemImpl implements o
}
}
}
- return new FileSystemImpl(fileSystems, "", conf);
+ return new VolumeManagerImpl(fileSystems, "", conf);
}
@Override
@@ -371,8 +371,8 @@ public class FileSystemImpl implements o
}
@Override
- public FileSystem getDefaultNamespace() {
- return namespaces.get(defaultNamespace);
+ public FileSystem getDefaultVolume() {
+ return volumes.get(defaultVolumes);
}
@Override
@@ -410,7 +410,7 @@ public class FileSystemImpl implements o
}
@Override
- public String newPathOnSameNamespace(String sourceDir, String suffix) {
+ public String newPathOnSameVolume(String sourceDir, String suffix) {
for (String fs : getFileSystems().keySet()) {
if (sourceDir.startsWith(fs)) {
return fs + "/" + suffix;
@@ -446,9 +446,8 @@ public class FileSystemImpl implements o
}
@Override
- public ContentSummary getContentSummary(String dir) {
- // TODO Auto-generated method stub
- return null;
+ public ContentSummary getContentSummary(Path dir) throws IOException {
+ return getFileSystemByPath(dir).getContentSummary(dir);
}
}
Propchange: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java Tue Jun 18 17:45:01 2013
@@ -39,7 +39,7 @@ import org.apache.accumulo.core.tabletse
import org.apache.accumulo.core.util.ThriftUtil;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.server.ServerConstants;
-import org.apache.accumulo.server.fs.FileSystem;
+import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.security.SecurityConstants;
import org.apache.accumulo.server.util.AddressUtil;
import org.apache.accumulo.server.util.MetadataTable;
@@ -58,11 +58,11 @@ public class GarbageCollectWriteAheadLog
private static final Logger log = Logger.getLogger(GarbageCollectWriteAheadLogs.class);
private final Instance instance;
- private final FileSystem fs;
+ private final VolumeManager fs;
private boolean useTrash;
- GarbageCollectWriteAheadLogs(Instance instance, FileSystem fs, boolean useTrash) throws IOException {
+ GarbageCollectWriteAheadLogs(Instance instance, VolumeManager fs, boolean useTrash) throws IOException {
this.instance = instance;
this.fs = fs;
}
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java Tue Jun 18 17:45:01 2013
@@ -76,8 +76,8 @@ import org.apache.accumulo.server.Accumu
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.ServerConfiguration;
-import org.apache.accumulo.server.fs.FileSystem;
-import org.apache.accumulo.server.fs.FileSystemImpl;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.master.state.tables.TableManager;
import org.apache.accumulo.server.security.SecurityConstants;
import org.apache.accumulo.server.util.Halt;
@@ -122,7 +122,7 @@ public class SimpleGarbageCollector impl
private TCredentials credentials;
private long gcStartDelay;
private boolean checkForBulkProcessingFiles;
- private FileSystem fs;
+ private VolumeManager fs;
private boolean useTrash = true;
private boolean safemode = false, offline = false, verbose = false;
private String address = "localhost";
@@ -140,7 +140,7 @@ public class SimpleGarbageCollector impl
Instance instance = HdfsZooInstance.getInstance();
ServerConfiguration serverConf = new ServerConfiguration(instance);
- final FileSystem fs = FileSystemImpl.get();
+ final VolumeManager fs = VolumeManagerImpl.get();
Accumulo.init(fs, serverConf, "gc");
String address = "localhost";
SimpleGarbageCollector gc = new SimpleGarbageCollector();
@@ -179,7 +179,7 @@ public class SimpleGarbageCollector impl
this.address = address;
}
- public void init(FileSystem fs, Instance instance, TCredentials credentials, boolean noTrash) throws IOException {
+ public void init(VolumeManager fs, Instance instance, TCredentials credentials, boolean noTrash) throws IOException {
this.fs = fs;
this.credentials = credentials;
this.instance = instance;
@@ -560,7 +560,6 @@ public class SimpleGarbageCollector impl
scanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
scanner.fetchColumnFamily(Constants.METADATA_SCANFILE_COLUMN_FAMILY);
Constants.METADATA_DIRECTORY_COLUMN.fetch(scanner);
- log.debug("Candidates: " + candidates);
TabletIterator tabletIterator = new TabletIterator(scanner, Constants.METADATA_KEYSPACE, false, true);
while (tabletIterator.hasNext()) {
@@ -583,7 +582,6 @@ public class SimpleGarbageCollector impl
delete = "/" + table + "/" + cf;
}
}
- log.debug("entry: " + delete);
// WARNING: This line is EXTREMELY IMPORTANT.
// You MUST REMOVE candidates that are still in use
if (candidates.remove(delete))
@@ -606,7 +604,7 @@ public class SimpleGarbageCollector impl
final static String METADATA_TABLE_DIR = "/" + Constants.METADATA_TABLE_ID;
private static void putMarkerDeleteMutation(final String delete, final BatchWriter writer, final BatchWriter rootWriter) throws MutationsRejectedException {
- if (delete.startsWith(METADATA_TABLE_DIR)) {
+ if (delete.contains(METADATA_TABLE_DIR)) {
Mutation m = new Mutation(new Text(Constants.METADATA_DELETE_FLAG_FOR_METADATA_PREFIX + delete));
m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
rootWriter.addMutation(m);
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java Tue Jun 18 17:45:01 2013
@@ -30,8 +30,8 @@ import java.util.regex.Pattern;
import org.apache.accumulo.core.cli.Help;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.server.fs.FileSystem;
-import org.apache.accumulo.server.fs.FileSystemImpl;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.tabletserver.log.DfsLogger;
import org.apache.accumulo.server.tabletserver.log.MultiReader;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -66,7 +66,7 @@ public class LogReader {
public static void main(String[] args) throws IOException {
Opts opts = new Opts();
opts.parseArgs(LogReader.class.getName(), args);
- FileSystem fs = FileSystemImpl.get();
+ VolumeManager fs = VolumeManagerImpl.get();
Matcher rowMatcher = null;
KeyExtent ke = null;
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/Master.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/Master.java Tue Jun 18 17:45:01 2013
@@ -98,8 +98,8 @@ import org.apache.accumulo.server.Accumu
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.fs.FileRef;
-import org.apache.accumulo.server.fs.FileSystem;
-import org.apache.accumulo.server.fs.FileSystemImpl;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
import org.apache.accumulo.server.master.balancer.DefaultLoadBalancer;
import org.apache.accumulo.server.master.balancer.TabletBalancer;
@@ -190,7 +190,7 @@ public class Master implements LiveTServ
final private static int MAX_TSERVER_WORK_CHUNK = 5000;
final private static int MAX_BAD_STATUS_COUNT = 3;
- final private FileSystem fs;
+ final private VolumeManager fs;
final private Instance instance;
final private String hostname;
final private LiveTServerSet tserverSet;
@@ -449,7 +449,7 @@ public class Master implements LiveTServ
return instance;
}
- public Master(ServerConfiguration config, FileSystem fs, String hostname) throws IOException {
+ public Master(ServerConfiguration config, VolumeManager fs, String hostname) throws IOException {
this.serverConfig = config;
this.instance = config.getInstance();
this.fs = fs;
@@ -2225,7 +2225,7 @@ public class Master implements LiveTServ
try {
SecurityUtil.serverLogin();
- FileSystem fs = FileSystemImpl.get();
+ VolumeManager fs = VolumeManagerImpl.get();
String hostname = Accumulo.getLocalAddress(args);
Instance instance = HdfsZooInstance.getInstance();
ServerConfiguration conf = new ServerConfiguration(instance);
@@ -2377,7 +2377,7 @@ public class Master implements LiveTServ
return serverConfig;
}
- public FileSystem getFileSystem() {
+ public VolumeManager getFileSystem() {
return this.fs;
}
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java Tue Jun 18 17:45:01 2013
@@ -21,7 +21,8 @@ import java.io.IOException;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.server.master.Master;
-import org.apache.accumulo.server.fs.FileSystem;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
@@ -32,8 +33,8 @@ public class HadoopLogCloser implements
private static Logger log = Logger.getLogger(HadoopLogCloser.class);
@Override
- public long close(Master master, FileSystem fs, Path source) throws IOException {
- org.apache.hadoop.fs.FileSystem ns = fs.getFileSystemByPath(source);
+ public long close(Master master, VolumeManager fs, Path source) throws IOException {
+ FileSystem ns = fs.getFileSystemByPath(source);
if (ns instanceof DistributedFileSystem) {
DistributedFileSystem dfs = (DistributedFileSystem) ns;
try {
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/LogCloser.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/LogCloser.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/LogCloser.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/LogCloser.java Tue Jun 18 17:45:01 2013
@@ -19,9 +19,9 @@ package org.apache.accumulo.server.maste
import java.io.IOException;
import org.apache.accumulo.server.master.Master;
-import org.apache.accumulo.server.fs.FileSystem;
+import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.hadoop.fs.Path;
public interface LogCloser {
- public long close(Master master, FileSystem fs, Path path) throws IOException;
+ public long close(Master master, VolumeManager fs, Path path) throws IOException;
}
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java Tue Jun 18 17:45:01 2013
@@ -19,7 +19,8 @@ package org.apache.accumulo.server.maste
import java.io.IOException;
import org.apache.accumulo.server.master.Master;
-import org.apache.accumulo.server.fs.FileSystem;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.log4j.Logger;
@@ -29,9 +30,9 @@ public class MapRLogCloser implements Lo
private static Logger log = Logger.getLogger(MapRLogCloser.class);
@Override
- public long close(Master m, FileSystem fs, Path path) throws IOException {
+ public long close(Master m, VolumeManager fs, Path path) throws IOException {
log.info("Recovering file " + path.toString() + " by changing permission to readonly");
- org.apache.hadoop.fs.FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getFileSystemByPath(path);
FsPermission roPerm = new FsPermission((short) 0444);
try {
ns.setPermission(path, roPerm);
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/state/SetGoalState.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/state/SetGoalState.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/state/SetGoalState.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/state/SetGoalState.java Tue Jun 18 17:45:01 2013
@@ -23,8 +23,8 @@ import org.apache.accumulo.core.zookeepe
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.accumulo.server.Accumulo;
import org.apache.accumulo.server.client.HdfsZooInstance;
-import org.apache.accumulo.server.fs.FileSystem;
-import org.apache.accumulo.server.fs.FileSystemImpl;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
public class SetGoalState {
@@ -39,7 +39,7 @@ public class SetGoalState {
}
SecurityUtil.serverLogin();
- FileSystem fs = FileSystemImpl.get();
+ VolumeManager fs = VolumeManagerImpl.get();
Accumulo.waitForZookeeperAndHdfs(fs);
ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZMASTER_GOAL_STATE, args[0].getBytes(),
NodeExistsPolicy.OVERWRITE);
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java Tue Jun 18 17:45:01 2013
@@ -62,7 +62,7 @@ import org.apache.accumulo.fate.Repo;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.ServerConfiguration;
-import org.apache.accumulo.server.fs.FileSystem;
+import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
import org.apache.accumulo.server.master.Master;
import org.apache.accumulo.server.master.state.TServerInstance;
@@ -146,7 +146,7 @@ public class BulkImport extends MasterRe
Utils.getReadLock(tableId, tid).lock();
// check that the error directory exists and is empty
- FileSystem fs = master.getFileSystem();
+ VolumeManager fs = master.getFileSystem();
Path errorPath = new Path(errorDir);
FileStatus errorStatus = null;
@@ -179,7 +179,7 @@ public class BulkImport extends MasterRe
}
}
- private Path createNewBulkDir(FileSystem fs, String tableId) throws IOException {
+ private Path createNewBulkDir(VolumeManager fs, String tableId) throws IOException {
String tableDir = null;
loop:
for (String dir : fs.getFileSystems().keySet()) {
@@ -218,7 +218,7 @@ public class BulkImport extends MasterRe
}
}
- private String prepareBulkImport(FileSystem fs, String dir, String tableId) throws IOException {
+ private String prepareBulkImport(VolumeManager fs, String dir, String tableId) throws IOException {
Path bulkDir = createNewBulkDir(fs, tableId);
MetadataTable.addBulkLoadInProgressFlag("/" + bulkDir.getParent().getName() + "/" + bulkDir.getName());
@@ -384,7 +384,7 @@ class CopyFailed extends MasterRepo {
public Repo<Master> call(long tid, Master master) throws Exception {
// This needs to execute after the arbiter is stopped
- FileSystem fs = master.getFileSystem();
+ VolumeManager fs = master.getFileSystem();
if (!fs.exists(new Path(error, BulkImport.FAILURES_TXT)))
return new CleanUpBulkImport(tableId, source, bulk, error);
@@ -505,7 +505,7 @@ class LoadFiles extends MasterRepo {
public Repo<Master> call(final long tid, final Master master) throws Exception {
initializeThreadPool(master);
final SiteConfiguration conf = ServerConfiguration.getSiteConfiguration();
- FileSystem fs = master.getFileSystem();
+ VolumeManager fs = master.getFileSystem();
List<FileStatus> files = new ArrayList<FileStatus>();
for (FileStatus entry : fs.listStatus(new Path(bulk))) {
files.add(entry);
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/CreateTable.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/CreateTable.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/CreateTable.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/CreateTable.java Tue Jun 18 17:45:01 2013
@@ -32,7 +32,7 @@ import org.apache.accumulo.core.security
import org.apache.accumulo.fate.Repo;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.accumulo.server.ServerConstants;
-import org.apache.accumulo.server.fs.FileSystem;
+import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.master.Master;
import org.apache.accumulo.server.master.state.tables.TableManager;
import org.apache.accumulo.server.security.AuditedSecurityOperation;
@@ -145,14 +145,14 @@ class CreateDir extends MasterRepo {
@Override
public Repo<Master> call(long tid, Master master) throws Exception {
- FileSystem fs = master.getFileSystem();
+ VolumeManager fs = master.getFileSystem();
TabletOperations.createTabletDirectory(fs, tableInfo.tableId, null);
return new PopulateMetadata(tableInfo);
}
@Override
public void undo(long tid, Master master) throws Exception {
- FileSystem fs = master.getFileSystem();
+ VolumeManager fs = master.getFileSystem();
for(String dir : ServerConstants.getTablesDirs()) {
fs.deleteRecursively(new Path(dir + "/" + tableInfo.tableId));
}
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/DeleteTable.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/DeleteTable.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/DeleteTable.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/DeleteTable.java Tue Jun 18 17:45:01 2013
@@ -37,7 +37,7 @@ import org.apache.accumulo.core.master.s
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.fate.Repo;
import org.apache.accumulo.server.ServerConstants;
-import org.apache.accumulo.server.fs.FileSystem;
+import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.master.Master;
import org.apache.accumulo.server.master.state.MetaDataTableScanner;
import org.apache.accumulo.server.master.state.TabletLocationState;
@@ -168,7 +168,7 @@ class CleanUp extends MasterRepo {
if (refCount == 0) {
// delete the map files
try {
- FileSystem fs = master.getFileSystem();
+ VolumeManager fs = master.getFileSystem();
for (String dir : ServerConstants.getTablesDirs()) {
fs.deleteRecursively(new Path(dir, tableId));
}
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/ExportTable.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/ExportTable.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/ExportTable.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/ExportTable.java Tue Jun 18 17:45:01 2013
@@ -49,7 +49,7 @@ import org.apache.accumulo.core.security
import org.apache.accumulo.fate.Repo;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.conf.TableConfiguration;
-import org.apache.accumulo.server.fs.FileSystem;
+import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.master.Master;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
@@ -138,7 +138,7 @@ class WriteExportFiles extends MasterRep
Utils.unreserveTable(tableInfo.tableID, tid, false);
}
- public static void exportTable(FileSystem fs, Connector conn, String tableName, String tableID, String exportDir) throws Exception {
+ public static void exportTable(VolumeManager fs, Connector conn, String tableName, String tableID, String exportDir) throws Exception {
fs.mkdirs(new Path(exportDir));
@@ -181,7 +181,7 @@ class WriteExportFiles extends MasterRep
}
}
- private static void createDistcpFile(FileSystem fs, String exportDir, Path exportMetaFilePath, Map<String,String> uniqueFiles) throws IOException {
+ private static void createDistcpFile(VolumeManager fs, String exportDir, Path exportMetaFilePath, Map<String,String> uniqueFiles) throws IOException {
BufferedWriter distcpOut = new BufferedWriter(new OutputStreamWriter(fs.create(new Path(exportDir, "distcp.txt"), false)));
try {
@@ -202,7 +202,7 @@ class WriteExportFiles extends MasterRep
}
}
- private static Map<String,String> exportMetadata(FileSystem fs, Connector conn, String tableID, ZipOutputStream zipOut, DataOutputStream dataOut) throws IOException,
+ private static Map<String,String> exportMetadata(VolumeManager fs, Connector conn, String tableID, ZipOutputStream zipOut, DataOutputStream dataOut) throws IOException,
TableNotFoundException {
zipOut.putNextEntry(new ZipEntry(Constants.EXPORT_METADATA_FILE));
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/ImportTable.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/ImportTable.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/ImportTable.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/tableOps/ImportTable.java Tue Jun 18 17:45:01 2013
@@ -52,7 +52,7 @@ import org.apache.accumulo.fate.Repo;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.client.HdfsZooInstance;
-import org.apache.accumulo.server.fs.FileSystem;
+import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.master.Master;
import org.apache.accumulo.server.master.state.tables.TableManager;
import org.apache.accumulo.server.security.AuditedSecurityOperation;
@@ -62,6 +62,7 @@ import org.apache.accumulo.server.tablet
import org.apache.accumulo.server.util.MetadataTable;
import org.apache.accumulo.server.util.TablePropUtil;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
@@ -136,7 +137,7 @@ class MoveExportedFiles extends MasterRe
@Override
public Repo<Master> call(long tid, Master master) throws Exception {
try {
- FileSystem fs = master.getFileSystem();
+ VolumeManager fs = master.getFileSystem();
Map<String,String> fileNameMappings = PopulateMetadataTable.readMappingFile(fs, tableInfo);
@@ -175,7 +176,7 @@ class PopulateMetadataTable extends Mast
this.tableInfo = ti;
}
- static Map<String,String> readMappingFile(FileSystem fs, ImportedTableInfo tableInfo) throws Exception {
+ static Map<String,String> readMappingFile(VolumeManager fs, ImportedTableInfo tableInfo) throws Exception {
BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(new Path(tableInfo.importDir, "mappings.txt"))));
try {
@@ -203,7 +204,7 @@ class PopulateMetadataTable extends Mast
ZipInputStream zis = null;
try {
- FileSystem fs = master.getFileSystem();
+ VolumeManager fs = master.getFileSystem();
mbw = master.getConnector().createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig());
@@ -311,7 +312,7 @@ class MapImportFileNames extends MasterR
BufferedWriter mappingsWriter = null;
try {
- FileSystem fs = environment.getFileSystem();
+ VolumeManager fs = environment.getFileSystem();
fs.mkdirs(new Path(tableInfo.importDir));
@@ -410,12 +411,12 @@ class ImportPopulateZookeeper extends Ma
return Utils.reserveTable(tableInfo.tableId, tid, true, false, TableOperation.IMPORT);
}
- private Map<String,String> getExportedProps(FileSystem fs) throws Exception {
+ private Map<String,String> getExportedProps(VolumeManager fs) throws Exception {
Path path = new Path(tableInfo.exportDir, Constants.EXPORT_FILE);
try {
- org.apache.hadoop.fs.FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getFileSystemByPath(path);
return TableOperationsImpl.getExportedProps(ns, path);
} catch (IOException ioe) {
throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/monitor/Monitor.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/monitor/Monitor.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/monitor/Monitor.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/monitor/Monitor.java Tue Jun 18 17:45:01 2013
@@ -51,8 +51,8 @@ import org.apache.accumulo.core.zookeepe
import org.apache.accumulo.server.Accumulo;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.ServerConfiguration;
-import org.apache.accumulo.server.fs.FileSystem;
-import org.apache.accumulo.server.fs.FileSystemImpl;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.monitor.servlets.DefaultServlet;
import org.apache.accumulo.server.monitor.servlets.GcStatusServlet;
import org.apache.accumulo.server.monitor.servlets.JSONServlet;
@@ -449,7 +449,7 @@ public class Monitor {
public static void main(String[] args) throws Exception {
SecurityUtil.serverLogin();
- FileSystem fs = FileSystemImpl.get();
+ VolumeManager fs = VolumeManagerImpl.get();
String hostname = Accumulo.getLocalAddress(args);
instance = HdfsZooInstance.getInstance();
config = new ServerConfiguration(instance);
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java Tue Jun 18 17:45:01 2013
@@ -54,7 +54,7 @@ import org.apache.accumulo.core.util.Loc
import org.apache.accumulo.core.util.MetadataTable.DataFileValue;
import org.apache.accumulo.server.conf.TableConfiguration;
import org.apache.accumulo.server.fs.FileRef;
-import org.apache.accumulo.server.fs.FileSystem;
+import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.problems.ProblemReport;
import org.apache.accumulo.server.problems.ProblemReportingIterator;
import org.apache.accumulo.server.problems.ProblemReports;
@@ -64,6 +64,7 @@ import org.apache.accumulo.server.tablet
import org.apache.accumulo.trace.instrument.Span;
import org.apache.accumulo.trace.instrument.Trace;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.log4j.Logger;
@@ -125,7 +126,7 @@ public class Compactor implements Callab
private TableConfiguration acuTableConf;
private CompactionEnv env;
private Configuration conf;
- private FileSystem fs;
+ private VolumeManager fs;
protected KeyExtent extent;
private List<IteratorSetting> iterators;
@@ -236,7 +237,7 @@ public class Compactor implements Callab
return compactions;
}
- Compactor(Configuration conf, FileSystem fs, Map<FileRef,DataFileValue> files, InMemoryMap imm, FileRef outputFile, boolean propogateDeletes,
+ Compactor(Configuration conf, VolumeManager fs, Map<FileRef,DataFileValue> files, InMemoryMap imm, FileRef outputFile, boolean propogateDeletes,
TableConfiguration acuTableConf, KeyExtent extent, CompactionEnv env, List<IteratorSetting> iterators, MajorCompactionReason reason) {
this.extent = extent;
this.conf = conf;
@@ -253,12 +254,12 @@ public class Compactor implements Callab
startTime = System.currentTimeMillis();
}
- Compactor(Configuration conf, FileSystem fs, Map<FileRef,DataFileValue> files, InMemoryMap imm, FileRef outputFile, boolean propogateDeletes,
+ Compactor(Configuration conf, VolumeManager fs, Map<FileRef,DataFileValue> files, InMemoryMap imm, FileRef outputFile, boolean propogateDeletes,
TableConfiguration acuTableConf, KeyExtent extent, CompactionEnv env) {
this(conf, fs, files, imm, outputFile, propogateDeletes, acuTableConf, extent, env, new ArrayList<IteratorSetting>(), null);
}
- public FileSystem getFileSystem() {
+ public VolumeManager getFileSystem() {
return fs;
}
@@ -283,7 +284,7 @@ public class Compactor implements Callab
try {
FileOperations fileFactory = FileOperations.getInstance();
- org.apache.hadoop.fs.FileSystem ns = this.fs.getFileSystemByPath(outputFile.path().toString());
+ FileSystem ns = this.fs.getFileSystemByPath(outputFile.path().toString());
mfw = fileFactory.openWriter(outputFile.path().toString(), ns, ns.getConf(), acuTableConf);
Map<String,Set<ByteSequence>> lGroups;
@@ -364,7 +365,7 @@ public class Compactor implements Callab
try {
FileOperations fileFactory = FileOperations.getInstance();
- org.apache.hadoop.fs.FileSystem fs = this.fs.getFileSystemByPath(mapFile.path().toString());
+ FileSystem fs = this.fs.getFileSystemByPath(mapFile.path().toString());
FileSKVIterator reader;
reader = fileFactory.openReader(mapFile.path().toString(), false, fs, conf, acuTableConf);
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/FileManager.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/FileManager.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/FileManager.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/FileManager.java Tue Jun 18 17:45:01 2013
@@ -45,12 +45,13 @@ import org.apache.accumulo.core.util.Met
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.fs.FileRef;
-import org.apache.accumulo.server.fs.FileSystem;
+import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.problems.ProblemReport;
import org.apache.accumulo.server.problems.ProblemReportingIterator;
import org.apache.accumulo.server.problems.ProblemReports;
import org.apache.accumulo.server.problems.ProblemType;
import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
@@ -102,7 +103,7 @@ public class FileManager {
private Semaphore filePermits;
- private FileSystem fs;
+ private VolumeManager fs;
// the data cache and index cache are allocated in
// TabletResourceManager and passed through the file opener to
@@ -161,7 +162,7 @@ public class FileManager {
* @param indexCache
* : underlying file can and should be able to handle a null cache
*/
- FileManager(ServerConfiguration conf, FileSystem fs, int maxOpen, BlockCache dataCache, BlockCache indexCache) {
+ FileManager(ServerConfiguration conf, VolumeManager fs, int maxOpen, BlockCache dataCache, BlockCache indexCache) {
if (maxOpen <= 0)
throw new IllegalArgumentException("maxOpen <= 0");
@@ -307,7 +308,7 @@ public class FileManager {
for (String file : filesToOpen) {
try {
Path path = fs.getFullPath(ServerConstants.getTablesDirs(), file);
- org.apache.hadoop.fs.FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getFileSystemByPath(path);
//log.debug("Opening "+file + " path " + path);
FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), false, ns, ns.getConf(), conf.getTableConfiguration(table.toString()),
dataCache, indexCache);
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/MinorCompactor.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/MinorCompactor.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/MinorCompactor.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/MinorCompactor.java Tue Jun 18 17:45:01 2013
@@ -36,7 +36,7 @@ import org.apache.accumulo.server.proble
import org.apache.accumulo.server.tabletserver.Tablet.MinorCompactionReason;
import org.apache.hadoop.conf.Configuration;
import org.apache.accumulo.server.fs.FileRef;
-import org.apache.accumulo.server.fs.FileSystem;
+import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;
@@ -53,7 +53,7 @@ public class MinorCompactor extends Comp
return Collections.singletonMap(mergeFile, dfv);
}
- MinorCompactor(Configuration conf, FileSystem fs, InMemoryMap imm, FileRef mergeFile, DataFileValue dfv, FileRef outputFile, TableConfiguration acuTableConf,
+ MinorCompactor(Configuration conf, VolumeManager fs, InMemoryMap imm, FileRef mergeFile, DataFileValue dfv, FileRef outputFile, TableConfiguration acuTableConf,
KeyExtent extent, MinorCompactionReason mincReason) {
super(conf, fs, toFileMap(mergeFile, dfv), imm, outputFile, true, acuTableConf, extent, new CompactionEnv() {
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java Tue Jun 18 17:45:01 2013
@@ -93,8 +93,8 @@ import org.apache.accumulo.server.client
import org.apache.accumulo.server.conf.TableConfiguration;
import org.apache.accumulo.server.constraints.ConstraintChecker;
import org.apache.accumulo.server.fs.FileRef;
-import org.apache.accumulo.server.fs.FileSystem;
-import org.apache.accumulo.server.fs.FileSystemImpl;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.master.state.TServerInstance;
import org.apache.accumulo.server.master.tableOps.CompactRange.CompactionIterators;
import org.apache.accumulo.server.problems.ProblemReport;
@@ -125,6 +125,7 @@ import org.apache.commons.codec.DecoderE
import org.apache.commons.codec.binary.Hex;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
@@ -386,7 +387,7 @@ public class Tablet {
private TServerInstance lastLocation;
private Configuration conf;
- private FileSystem fs;
+ private VolumeManager fs;
private TableConfiguration acuTableConf;
@@ -1060,7 +1061,7 @@ public class Tablet {
private Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, Configuration conf,
SortedMap<Key,Value> tabletsKeyValues) throws IOException {
- this(tabletServer, location, extent, trm, conf, FileSystemImpl.get(),
+ this(tabletServer, location, extent, trm, conf, VolumeManagerImpl.get(),
tabletsKeyValues);
}
@@ -1068,7 +1069,7 @@ public class Tablet {
private Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, Configuration conf,
SortedMap<FileRef,DataFileValue> datafiles, String time, long initFlushID, long initCompactID) throws IOException {
- this(tabletServer, location, extent, trm, conf, FileSystemImpl.get(), EMPTY,
+ this(tabletServer, location, extent, trm, conf, VolumeManagerImpl.get(), EMPTY,
datafiles, time, null, new HashSet<FileRef>(), initFlushID, initCompactID);
}
@@ -1094,14 +1095,14 @@ public class Tablet {
return null;
}
- private static SortedMap<FileRef,DataFileValue> lookupDatafiles(AccumuloConfiguration conf, Text locText, FileSystem fs, KeyExtent extent,
+ private static SortedMap<FileRef,DataFileValue> lookupDatafiles(AccumuloConfiguration conf, Text locText, VolumeManager fs, KeyExtent extent,
SortedMap<Key,Value> tabletsKeyValues) throws IOException {
TreeMap<FileRef,DataFileValue> datafiles = new TreeMap<FileRef,DataFileValue>();
if (extent.isRootTablet()) { // the meta0 tablet
Path location = new Path(ServerConstants.getRootTabletDir());
- location = location.makeQualified(fs.getDefaultNamespace());
+ location = location.makeQualified(fs.getDefaultVolume());
// cleanUpFiles() has special handling for delete. files
FileStatus[] files = fs.listStatus(location);
Collection<String> goodPaths = cleanUpFiles(fs, files, true);
@@ -1168,7 +1169,7 @@ public class Tablet {
return logEntries;
}
- private static Set<FileRef> lookupScanFiles(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues, FileSystem fs) throws IOException {
+ private static Set<FileRef> lookupScanFiles(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues, VolumeManager fs) throws IOException {
HashSet<FileRef> scanFiles = new HashSet<FileRef>();
Text row = extent.getMetadataEntry();
@@ -1206,7 +1207,7 @@ public class Tablet {
return -1;
}
- private Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, Configuration conf, FileSystem fs,
+ private Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, Configuration conf, VolumeManager fs,
SortedMap<Key,Value> tabletsKeyValues) throws IOException {
this(tabletServer, location, extent, trm, conf, fs, lookupLogEntries(extent, tabletsKeyValues), lookupDatafiles(tabletServer.getSystemConfiguration(),
location, fs, extent, tabletsKeyValues), lookupTime(tabletServer.getSystemConfiguration(), extent, tabletsKeyValues), lookupLastServer(extent,
@@ -1226,7 +1227,7 @@ public class Tablet {
* yet another constructor - this one allows us to avoid costly lookups into the Metadata table if we already know the files we need - as at split time
*/
private Tablet(final TabletServer tabletServer, final Text location, final KeyExtent extent, final TabletResourceManager trm, final Configuration conf,
- final FileSystem fs, final List<LogEntry> logEntries, final SortedMap<FileRef,DataFileValue> datafiles, String time, final TServerInstance lastLocation,
+ final VolumeManager fs, final List<LogEntry> logEntries, final SortedMap<FileRef,DataFileValue> datafiles, String time, final TServerInstance lastLocation,
Set<FileRef> scanFiles, long initFlushID, long initCompactID) throws IOException {
if (location.find(":") >= 0) {
this.location = new Path(location.toString());
@@ -1251,7 +1252,7 @@ public class Tablet {
long rtime = Long.MIN_VALUE;
for (FileRef ref : datafiles.keySet()) {
Path path = ref.path();
- org.apache.hadoop.fs.FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getFileSystemByPath(path);
FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), true, ns, ns.getConf(),
tabletServer.getTableConfiguration(extent));
long maxTime = -1;
@@ -1419,7 +1420,7 @@ public class Tablet {
}
}
- private static Collection<String> cleanUpFiles(FileSystem fs, FileStatus[] files, boolean deleteTmp) throws IOException {
+ private static Collection<String> cleanUpFiles(VolumeManager fs, FileStatus[] files, boolean deleteTmp) throws IOException {
/*
* called in constructor and before major compactions
*/
@@ -2040,7 +2041,7 @@ public class Tablet {
}
- private DataFileValue minorCompact(Configuration conf, FileSystem fs, InMemoryMap memTable, FileRef tmpDatafile, FileRef newDatafile, FileRef mergeFile,
+ private DataFileValue minorCompact(Configuration conf, VolumeManager fs, InMemoryMap memTable, FileRef tmpDatafile, FileRef newDatafile, FileRef mergeFile,
boolean hasQueueTime, long queued, CommitSession commitSession, long flushId, MinorCompactionReason mincReason) {
boolean failed = false;
long start = System.currentTimeMillis();
@@ -2906,7 +2907,7 @@ public class Tablet {
for (Entry<FileRef,DataFileValue> entry : files.entrySet()) {
FileRef file = entry.getKey();
- org.apache.hadoop.fs.FileSystem ns = fs.getFileSystemByPath(file.path());
+ FileSystem ns = fs.getFileSystemByPath(file.path());
FileSKVIterator openReader = fileFactory.openReader(file.path().toString(), true, ns, ns.getConf(), acuTableConf);
try {
Key first = openReader.getFirstKey();
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java Tue Jun 18 17:45:01 2013
@@ -140,8 +140,8 @@ import org.apache.accumulo.server.conf.S
import org.apache.accumulo.server.conf.TableConfiguration;
import org.apache.accumulo.server.data.ServerMutation;
import org.apache.accumulo.server.fs.FileRef;
-import org.apache.accumulo.server.fs.FileSystem;
-import org.apache.accumulo.server.fs.FileSystemImpl;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.master.state.Assignment;
import org.apache.accumulo.server.master.state.DistributedStoreException;
import org.apache.accumulo.server.master.state.TServerInstance;
@@ -201,6 +201,7 @@ import org.apache.accumulo.trace.instrum
import org.apache.accumulo.trace.instrument.thrift.TraceWrap;
import org.apache.accumulo.trace.thrift.TInfo;
import org.apache.commons.collections.map.LRUMap;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
@@ -231,7 +232,7 @@ public class TabletServer extends Abstra
private ServerConfiguration serverConfig;
private LogSorter logSorter = null;
- public TabletServer(ServerConfiguration conf, FileSystem fs) {
+ public TabletServer(ServerConfiguration conf, VolumeManager fs) {
super();
this.serverConfig = conf;
this.instance = conf.getInstance();
@@ -881,7 +882,7 @@ public class TabletServer extends Abstra
Map<String,MapFileInfo> fileMap = entry.getValue();
Map<FileRef, MapFileInfo> fileRefMap = new HashMap<FileRef, MapFileInfo>();
for (Entry<String,MapFileInfo> mapping : fileMap.entrySet()) {
- org.apache.hadoop.fs.FileSystem ns = fs.getFileSystemByPath(mapping.getKey());
+ FileSystem ns = fs.getFileSystemByPath(mapping.getKey());
Path path = ns.makeQualified(new Path(mapping.getKey()));
fileRefMap.put(new FileRef(path.toString(), path), mapping.getValue());
}
@@ -2549,7 +2550,7 @@ public class TabletServer extends Abstra
}
}
- private FileSystem fs;
+ private VolumeManager fs;
private Instance instance;
private final SortedMap<KeyExtent,Tablet> onlineTablets = Collections.synchronizedSortedMap(new TreeMap<KeyExtent,Tablet>());
@@ -3217,7 +3218,7 @@ public class TabletServer extends Abstra
public static void main(String[] args) throws IOException {
try {
SecurityUtil.serverLogin();
- FileSystem fs = FileSystemImpl.get();
+ VolumeManager fs = VolumeManagerImpl.get();
String hostname = Accumulo.getLocalAddress(args);
Instance instance = HdfsZooInstance.getInstance();
ServerConfiguration conf = new ServerConfiguration(instance);
@@ -3240,7 +3241,7 @@ public class TabletServer extends Abstra
logger.minorCompactionStarted(tablet, lastUpdateSequence, newMapfileLocation);
}
- public void recover(FileSystem fs, Tablet tablet, List<LogEntry> logEntries, Set<String> tabletFiles, MutationReceiver mutationReceiver) throws IOException {
+ public void recover(VolumeManager fs, Tablet tablet, List<LogEntry> logEntries, Set<String> tabletFiles, MutationReceiver mutationReceiver) throws IOException {
List<Path> recoveryLogs = new ArrayList<Path>();
List<LogEntry> sorted = new ArrayList<LogEntry>(logEntries);
Collections.sort(sorted, new Comparator<LogEntry>() {
@@ -3455,7 +3456,7 @@ public class TabletServer extends Abstra
return new DfsLogger.ServerResources() {
@Override
- public FileSystem getFileSystem() {
+ public VolumeManager getFileSystem() {
return fs;
}
@@ -3471,7 +3472,7 @@ public class TabletServer extends Abstra
};
}
- public FileSystem getFileSystem() {
+ public VolumeManager getFileSystem() {
return fs;
}
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java Tue Jun 18 17:45:01 2013
@@ -49,7 +49,7 @@ import org.apache.accumulo.core.util.Nam
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.fs.FileRef;
-import org.apache.accumulo.server.fs.FileSystem;
+import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.tabletserver.FileManager.ScanFileManager;
import org.apache.accumulo.server.tabletserver.Tablet.MajorCompactionReason;
import org.apache.accumulo.server.tabletserver.Tablet.MinorCompactionReason;
@@ -141,7 +141,7 @@ public class TabletServerResourceManager
return addEs(name, new ThreadPoolExecutor(min, max, timeout, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new NamingThreadFactory(name)));
}
- public TabletServerResourceManager(Instance instance, FileSystem fs) {
+ public TabletServerResourceManager(Instance instance, VolumeManager fs) {
this.conf = new ServerConfiguration(instance);
final AccumuloConfiguration acuConf = conf.getConfiguration();
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java Tue Jun 18 17:45:01 2013
@@ -44,7 +44,7 @@ import org.apache.accumulo.core.data.Mut
import org.apache.accumulo.core.util.Daemon;
import org.apache.accumulo.core.util.StringUtil;
import org.apache.accumulo.server.ServerConstants;
-import org.apache.accumulo.server.fs.FileSystem;
+import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.logger.LogFileKey;
import org.apache.accumulo.server.logger.LogFileValue;
import org.apache.accumulo.server.master.state.TServerInstance;
@@ -75,7 +75,7 @@ public class DfsLogger {
public interface ServerResources {
AccumuloConfiguration getConfiguration();
- FileSystem getFileSystem();
+ VolumeManager getFileSystem();
Set<TServerInstance> getCurrentTServers();
}
@@ -208,7 +208,7 @@ public class DfsLogger {
this.logPath = filename;
}
- public static FSDataInputStream readHeader(FileSystem fs, Path path, Map<String,String> opts) throws IOException {
+ public static FSDataInputStream readHeader(VolumeManager fs, Path path, Map<String,String> opts) throws IOException {
FSDataInputStream file = fs.open(path);
try {
byte[] magic = LOG_FILE_HEADER_V2.getBytes();
@@ -244,7 +244,7 @@ public class DfsLogger {
logPath = new Path(wals[random.nextInt(wals.length)] + "/" + logger + "/" + filename);
try {
- FileSystem fs = conf.getFileSystem();
+ VolumeManager fs = conf.getFileSystem();
short replication = (short) conf.getConfiguration().getCount(Property.TSERV_WAL_REPLICATION);
if (replication == 0)
replication = fs.getDefaultReplication(logPath);
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java Tue Jun 18 17:45:01 2013
@@ -37,12 +37,13 @@ import org.apache.accumulo.core.master.t
import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.core.util.SimpleThreadPool;
import org.apache.accumulo.core.zookeeper.ZooUtil;
-import org.apache.accumulo.server.fs.FileSystem;
+import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.logger.LogFileKey;
import org.apache.accumulo.server.logger.LogFileValue;
import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
import org.apache.accumulo.server.zookeeper.DistributedWorkQueue.Processor;
import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.MapFile;
import org.apache.log4j.Logger;
@@ -54,7 +55,7 @@ import org.apache.zookeeper.KeeperExcept
public class LogSorter {
private static final Logger log = Logger.getLogger(LogSorter.class);
- FileSystem fs;
+ VolumeManager fs;
AccumuloConfiguration conf;
private final Map<String,LogProcessor> currentWork = Collections.synchronizedMap(new HashMap<String,LogProcessor>());
@@ -196,7 +197,7 @@ public class LogSorter {
private void writeBuffer(String destPath, ArrayList<Pair<LogFileKey,LogFileValue>> buffer, int part) throws IOException {
String path = destPath + String.format("/part-r-%05d", part++);
- org.apache.hadoop.fs.FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getFileSystemByPath(path);
MapFile.Writer output = new MapFile.Writer(ns.getConf(), ns, path, LogFileKey.class, LogFileValue.class);
try {
Collections.sort(buffer, new Comparator<Pair<LogFileKey,LogFileValue>>() {
@@ -237,7 +238,7 @@ public class LogSorter {
ThreadPoolExecutor threadPool;
private final Instance instance;
- public LogSorter(Instance instance, FileSystem fs, AccumuloConfiguration conf) {
+ public LogSorter(Instance instance, VolumeManager fs, AccumuloConfiguration conf) {
this.instance = instance;
this.fs = fs;
this.conf = conf;
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/MultiReader.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/MultiReader.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/MultiReader.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/MultiReader.java Tue Jun 18 17:45:01 2013
@@ -19,9 +19,10 @@ package org.apache.accumulo.server.table
import java.io.EOFException;
import java.io.IOException;
-import org.apache.accumulo.server.fs.FileSystem;
+import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.commons.collections.buffer.PriorityBuffer;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
@@ -86,7 +87,7 @@ public class MultiReader {
private PriorityBuffer heap = new PriorityBuffer();
- public MultiReader(FileSystem fs, Path directory) throws IOException {
+ public MultiReader(VolumeManager fs, Path directory) throws IOException {
boolean foundFinish = false;
for (FileStatus child : fs.listStatus(directory)) {
if (child.getPath().getName().startsWith("_"))
@@ -95,7 +96,7 @@ public class MultiReader {
foundFinish = true;
continue;
}
- org.apache.hadoop.fs.FileSystem ns = fs.getFileSystemByPath(child.getPath());
+ FileSystem ns = fs.getFileSystemByPath(child.getPath());
heap.add(new Index(new Reader(ns, child.getPath().toString(), ns.getConf())));
}
if (!foundFinish)
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecovery.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecovery.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecovery.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecovery.java Tue Jun 18 17:45:01 2013
@@ -29,7 +29,7 @@ import java.util.Set;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.server.fs.FileSystem;
+import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.logger.LogFileKey;
import org.apache.accumulo.server.logger.LogFileValue;
import org.apache.hadoop.fs.Path;
@@ -54,9 +54,9 @@ public class SortedLogRecovery {
public UnusedException() { super(); }
}
- private FileSystem fs;
+ private VolumeManager fs;
- public SortedLogRecovery(FileSystem fs) {
+ public SortedLogRecovery(VolumeManager fs) {
this.fs = fs;
}
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/TabletServerLogger.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/TabletServerLogger.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/TabletServerLogger.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/TabletServerLogger.java Tue Jun 18 17:45:01 2013
@@ -34,7 +34,7 @@ import org.apache.accumulo.core.conf.Pro
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.accumulo.server.fs.FileSystem;
+import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.tabletserver.Tablet;
import org.apache.accumulo.server.tabletserver.Tablet.CommitSession;
import org.apache.accumulo.server.tabletserver.TabletMutations;
@@ -415,7 +415,7 @@ public class TabletServerLogger {
return seq;
}
- public void recover(FileSystem fs, Tablet tablet, List<Path> logs, Set<String> tabletFiles, MutationReceiver mr) throws IOException {
+ public void recover(VolumeManager fs, Tablet tablet, List<Path> logs, Set<String> tabletFiles, MutationReceiver mr) throws IOException {
if (!enabled(tablet))
return;
try {
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java Tue Jun 18 17:45:01 2013
@@ -45,8 +45,8 @@ import org.apache.accumulo.fate.zookeepe
import org.apache.accumulo.server.Accumulo;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.ServerConfiguration;
-import org.apache.accumulo.server.fs.FileSystem;
-import org.apache.accumulo.server.fs.FileSystemImpl;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.util.time.SimpleTimer;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
import org.apache.accumulo.start.classloader.AccumuloClassLoader;
@@ -253,7 +253,7 @@ public class TraceServer implements Watc
SecurityUtil.serverLogin();
Instance instance = HdfsZooInstance.getInstance();
ServerConfiguration conf = new ServerConfiguration(instance);
- FileSystem fs = FileSystemImpl.get();
+ VolumeManager fs = VolumeManagerImpl.get();
Accumulo.init(fs, conf, "tracer");
String hostname = Accumulo.getLocalAddress(args);
TraceServer server = new TraceServer(conf, hostname);
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/CheckForMetadataProblems.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/CheckForMetadataProblems.java?rev=1494226&r1=1494225&r2=1494226&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/CheckForMetadataProblems.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/CheckForMetadataProblems.java Tue Jun 18 17:45:01 2013
@@ -36,8 +36,8 @@ import org.apache.accumulo.core.security
import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.accumulo.server.conf.ServerConfiguration;
-import org.apache.accumulo.server.fs.FileSystem;
-import org.apache.accumulo.server.fs.FileSystemImpl;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.hadoop.io.Text;
import com.beust.jcommander.Parameter;
@@ -98,7 +98,7 @@ public class CheckForMetadataProblems {
sawProblems = true;
}
- public static void checkMetadataTableEntries(Opts opts, FileSystem fs) throws Exception {
+ public static void checkMetadataTableEntries(Opts opts, VolumeManager fs) throws Exception {
Map<String,TreeSet<KeyExtent>> tables = new HashMap<String,TreeSet<KeyExtent>>();
Scanner scanner;
@@ -192,7 +192,7 @@ public class CheckForMetadataProblems {
Opts opts = new Opts();
opts.parseArgs(CheckForMetadataProblems.class.getName(), args);
- FileSystem fs = FileSystemImpl.get();
+ VolumeManager fs = VolumeManagerImpl.get();
checkMetadataTableEntries(opts, fs);
opts.stopTracing();
if (sawProblems)