You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/03/11 19:25:13 UTC
[04/23] git commit: ACCUMULO-2061 Fix the rest of server-base
ACCUMULO-2061 Fix the rest of server-base
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e613f710
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e613f710
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e613f710
Branch: refs/heads/ACCUMULO-2061
Commit: e613f7105e3a0828096909535147b4bc80a265f6
Parents: 3b3ccbb
Author: Josh Elser <el...@apache.org>
Authored: Fri Mar 7 17:14:17 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Fri Mar 7 17:14:17 2014 -0500
----------------------------------------------------------------------
.../apache/accumulo/server/client/BulkImporter.java | 4 ++--
.../accumulo/server/client/HdfsZooInstance.java | 15 ++++++++++++++-
.../accumulo/server/conf/ZooConfiguration.java | 16 +++++++++++++++-
.../org/apache/accumulo/server/fs/VolumeUtil.java | 10 ++--------
.../org/apache/accumulo/server/init/Initialize.java | 2 +-
.../server/master/recovery/HadoopLogCloser.java | 2 +-
.../server/master/recovery/MapRLogCloser.java | 2 +-
.../apache/accumulo/server/util/LocalityCheck.java | 2 +-
.../accumulo/server/util/TabletOperations.java | 4 ++--
9 files changed, 39 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java b/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
index dc9acf8..27ab078 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
@@ -339,7 +339,7 @@ public class BulkImporter {
try {
for (Path path : paths) {
- FileSystem fs = vm.getFileSystemByPath(path);
+ FileSystem fs = vm.getVolumeByPath(path).getFileSystem();
mapFileSizes.put(path, fs.getContentSummary(path).getLength());
}
} catch (IOException e) {
@@ -639,7 +639,7 @@ public class BulkImporter {
Collection<ByteSequence> columnFamilies = Collections.emptyList();
String filename = file.toString();
// log.debug(filename + " finding overlapping tablets " + startRow + " -> " + endRow);
- FileSystem fs = vm.getFileSystemByPath(file);
+ FileSystem fs = vm.getVolumeByPath(file).getFileSystem();
FileSKVIterator reader = FileOperations.getInstance().openReader(filename, true, fs, fs.getConf(), acuConf);
try {
Text row = startRow;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
index 6993a0a..8aa0be9 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.server.client;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.List;
@@ -38,10 +39,13 @@ import org.apache.accumulo.core.util.ByteBufferUtil;
import org.apache.accumulo.core.util.OpTimer;
import org.apache.accumulo.core.util.StringUtil;
import org.apache.accumulo.core.util.TextUtil;
+import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.ZooCache;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.zookeeper.ZooLock;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Level;
@@ -122,7 +126,16 @@ public class HdfsZooInstance implements Instance {
private static synchronized void _getInstanceID() {
if (instanceId == null) {
- String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(), ServerConfiguration.getSiteConfiguration());
+ AccumuloConfiguration acuConf = ServerConfiguration.getSiteConfiguration();
+ // InstanceID should be the same across all volumes, so just choose one
+ VolumeManager fs;
+ try {
+ fs = VolumeManagerImpl.get();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ Volume randVolume = fs.getVolumes().iterator().next();
+ String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(randVolume), acuConf);
instanceId = instanceIdFromFile;
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
index 32f6126..94e468b 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.server.conf;
+import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -25,10 +26,15 @@ import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.volume.Volume;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.ZooCache;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.client.HdfsZooInstance.AccumuloNotInitializedException;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.log4j.Logger;
public class ZooConfiguration extends AccumuloConfiguration {
@@ -57,7 +63,15 @@ public class ZooConfiguration extends AccumuloConfiguration {
if (instance == null) {
propCache = new ZooCache(parent.get(Property.INSTANCE_ZK_HOST), (int) parent.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT));
instance = new ZooConfiguration(parent);
- String deprecatedInstanceIdFromHdfs = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(), parent);
+ // InstanceID should be the same across all volumes, so just choose one
+ VolumeManager fs;
+ try {
+ fs = VolumeManagerImpl.get();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ Volume randVolume = fs.getVolumes().iterator().next();
+ String deprecatedInstanceIdFromHdfs = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(randVolume), parent);
instanceId = deprecatedInstanceIdFromHdfs;
}
return instance;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
index 42ea912..110f1e4 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
@@ -26,21 +26,17 @@ import java.util.Map.Entry;
import java.util.SortedMap;
import java.util.TreeMap;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.tabletserver.log.LogEntry;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.Pair;
-import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.fs.VolumeManager.FileType;
import org.apache.accumulo.server.security.SystemCredentials;
import org.apache.accumulo.server.util.MetadataTableUtil;
import org.apache.accumulo.server.zookeeper.ZooLock;
import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -49,8 +45,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
-import com.google.common.base.Preconditions;
-
/**
* Utility methods for managing absolute URIs contained in Accumulo metadata.
*/
@@ -245,8 +239,8 @@ public class VolumeUtil {
// this code needs to be idempotent
- FileSystem fs1 = vm.getFileSystemByPath(dir);
- FileSystem fs2 = vm.getFileSystemByPath(newDir);
+ FileSystem fs1 = vm.getVolumeByPath(dir).getFileSystem();
+ FileSystem fs2 = vm.getVolumeByPath(newDir).getFileSystem();
if (!same(fs1, dir, fs2, newDir)) {
if (fs2.exists(newDir)) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
index 59fad73..206a721 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
@@ -341,7 +341,7 @@ public class Initialize {
// the root tablet contains the key extent and locations of all the
// metadata tablets
String initRootTabFile = rootTablet + "/00000_00000." + FileOperations.getNewFileExtension(AccumuloConfiguration.getDefaultConfiguration());
- FileSystem ns = fs.getFileSystemByPath(new Path(initRootTabFile));
+ FileSystem ns = fs.getVolumeByPath(new Path(initRootTabFile)).getFileSystem();
FileSKVWriter mfw = FileOperations.getInstance().openWriter(initRootTabFile, ns, ns.getConf(), AccumuloConfiguration.getDefaultConfiguration());
mfw.startDefaultLocalityGroup();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
index 7edc0cf..0e7f83a 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
@@ -37,7 +37,7 @@ public class HadoopLogCloser implements LogCloser {
@Override
public long close(AccumuloConfiguration conf, VolumeManager fs, Path source) throws IOException {
- FileSystem ns = fs.getFileSystemByPath(source);
+ FileSystem ns = fs.getVolumeByPath(source).getFileSystem();
// if path points to a viewfs path, then resolve to underlying filesystem
if (ViewFSUtils.isViewFS(ns)) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java
index bba7ac5..d737109 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java
@@ -32,7 +32,7 @@ public class MapRLogCloser implements LogCloser {
@Override
public long close(AccumuloConfiguration conf, VolumeManager fs, Path path) throws IOException {
log.info("Recovering file " + path.toString() + " by changing permission to readonly");
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
FsPermission roPerm = new FsPermission((short) 0444);
try {
ns.setPermission(path, roPerm);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java b/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
index a96e791..35bb8f5 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
@@ -88,7 +88,7 @@ public class LocalityCheck {
}
for (String file : files) {
Path filePath = new Path(file);
- FileSystem ns = fs.getFileSystemByPath(filePath);
+ FileSystem ns = fs.getVolumeByPath(filePath).getFileSystem();
FileStatus fileStatus = ns.getFileStatus(filePath);
BlockLocation[] fileBlockLocations = ns.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
for (BlockLocation blockLocation : fileBlockLocations) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java b/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
index b237cd0..b8e7113 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
@@ -46,7 +46,7 @@ public class TabletOperations {
lowDirectory = Constants.DEFAULT_TABLET_LOCATION;
Path lowDirectoryPath = new Path(volume + "/" + tableId + "/" + lowDirectory);
if (fs.exists(lowDirectoryPath) || fs.mkdirs(lowDirectoryPath)) {
- FileSystem pathFs = fs.getFileSystemByPath(lowDirectoryPath);
+ FileSystem pathFs = fs.getVolumeByPath(lowDirectoryPath).getFileSystem();
return lowDirectoryPath.makeQualified(pathFs.getUri(), pathFs.getWorkingDirectory()).toString();
}
log.warn("Failed to create " + lowDirectoryPath + " for unknown reason");
@@ -56,7 +56,7 @@ public class TabletOperations {
if (fs.exists(lowDirectoryPath))
throw new IllegalStateException("Dir exist when it should not " + lowDirectoryPath);
if (fs.mkdirs(lowDirectoryPath)) {
- FileSystem lowDirectoryFs = fs.getFileSystemByPath(lowDirectoryPath);
+ FileSystem lowDirectoryFs = fs.getVolumeByPath(lowDirectoryPath).getFileSystem();
return lowDirectoryPath.makeQualified(lowDirectoryFs.getUri(), lowDirectoryFs.getWorkingDirectory()).toString();
}
}