You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2015/08/18 02:40:49 UTC
[2/2] hadoop git commit: HDFS-8895. Remove deprecated
BlockStorageLocation APIs.
HDFS-8895. Remove deprecated BlockStorageLocation APIs.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eee4d716
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eee4d716
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eee4d716
Branch: refs/heads/trunk
Commit: eee4d716b48074825e1afcd9c74038a393ddeb69
Parents: a88f31e
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Aug 17 17:40:37 2015 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Mon Aug 17 17:40:37 2015 -0700
----------------------------------------------------------------------
.../src/main/proto/ClientDatanodeProtocol.proto | 33 --
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../apache/hadoop/fs/BlockStorageLocation.java | 52 ---
.../java/org/apache/hadoop/fs/HdfsVolumeId.java | 73 ----
.../java/org/apache/hadoop/fs/VolumeId.java | 40 --
.../hadoop/hdfs/BlockStorageLocationUtil.java | 368 -------------------
.../java/org/apache/hadoop/hdfs/DFSClient.java | 85 -----
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 -
.../hadoop/hdfs/DistributedFileSystem.java | 39 --
.../apache/hadoop/hdfs/HdfsConfiguration.java | 2 -
.../hadoop/hdfs/client/impl/DfsClientConf.java | 33 --
.../hdfs/protocol/ClientDatanodeProtocol.java | 19 -
.../hdfs/protocol/HdfsBlocksMetadata.java | 111 ------
...tDatanodeProtocolServerSideTranslatorPB.java | 41 ---
.../ClientDatanodeProtocolTranslatorPB.java | 43 ---
.../hadoop/hdfs/server/datanode/DataNode.java | 29 --
.../server/datanode/fsdataset/FsDatasetSpi.java | 13 -
.../datanode/fsdataset/impl/FsDatasetImpl.java | 45 ---
.../src/main/resources/hdfs-default.xml | 24 --
.../java/org/apache/hadoop/fs/TestVolumeId.java | 146 --------
.../hadoop/hdfs/TestDistributedFileSystem.java | 230 ------------
.../server/datanode/SimulatedFSDataset.java | 7 -
.../extdataset/ExternalDatasetImpl.java | 6 -
23 files changed, 2 insertions(+), 1445 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eee4d716/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto
index e0d1f5f..6824b48 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto
@@ -102,32 +102,6 @@ message GetBlockLocalPathInfoResponseProto {
}
/**
- * Query for the disk locations of a number of blocks on this DN.
- * blockPoolId - the pool to query
- * blockIds - list of block IDs to query
- * tokens - list of access tokens corresponding to list of block IDs
- */
-message GetHdfsBlockLocationsRequestProto {
- // Removed: HDFS-3969
- // repeated ExtendedBlockProto blocks = 1;
- repeated hadoop.common.TokenProto tokens = 2;
-
- required string blockPoolId = 3;
- repeated sfixed64 blockIds = 4 [ packed = true ];
-}
-
-/**
- * volumeIds - id of each volume, potentially multiple bytes
- * volumeIndexes - for each block, an index into volumeIds specifying the volume
- * on which it is located. If block is not present on any volume,
- * index is set to MAX_INT.
- */
-message GetHdfsBlockLocationsResponseProto {
- repeated bytes volumeIds = 1;
- repeated uint32 volumeIndexes = 2 [ packed = true ];
-}
-
-/**
* forUpgrade - if true, clients are advised to wait for restart and quick
* upgrade restart is instrumented. Otherwise, datanode does
* the regular shutdown.
@@ -219,13 +193,6 @@ service ClientDatanodeProtocolService {
rpc getBlockLocalPathInfo(GetBlockLocalPathInfoRequestProto)
returns(GetBlockLocalPathInfoResponseProto);
- /**
- * Retrieve additional HDFS-specific metadata about a set of blocks stored
- * on the local file system.
- */
- rpc getHdfsBlockLocations(GetHdfsBlockLocationsRequestProto)
- returns(GetHdfsBlockLocationsResponseProto);
-
rpc shutdownDatanode(ShutdownDatanodeRequestProto)
returns(ShutdownDatanodeResponseProto);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eee4d716/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 944e165..7d05bc2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -34,6 +34,8 @@ Trunk (Unreleased)
HDFS-8591. Remove support for deprecated configuration key
dfs.namenode.decommission.nodes.per.interval. (wang)
+ HDFS-8895. Remove deprecated BlockStorageLocation APIs. (wang)
+
NEW FEATURES
HDFS-3125. Add JournalService to enable Journal Daemon. (suresh)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eee4d716/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/BlockStorageLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/BlockStorageLocation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/BlockStorageLocation.java
deleted file mode 100644
index 2200994..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/BlockStorageLocation.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * Wrapper for {@link BlockLocation} that also adds {@link VolumeId} volume
- * location information for each replica.
- */
-@InterfaceStability.Unstable
-@InterfaceAudience.Public
-@Deprecated
-public class BlockStorageLocation extends BlockLocation {
-
- private final VolumeId[] volumeIds;
-
- public BlockStorageLocation(BlockLocation loc, VolumeId[] volumeIds)
- throws IOException {
- // Initialize with data from passed in BlockLocation
- super(loc.getNames(), loc.getHosts(), loc.getTopologyPaths(), loc
- .getOffset(), loc.getLength(), loc.isCorrupt());
- this.volumeIds = volumeIds;
- }
-
- /**
- * Gets the list of {@link VolumeId} corresponding to the block's replicas.
- *
- * @return volumeIds list of VolumeId for the block's replicas
- */
- public VolumeId[] getVolumeIds() {
- return volumeIds;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eee4d716/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/HdfsVolumeId.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/HdfsVolumeId.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/HdfsVolumeId.java
deleted file mode 100644
index 6e9d3d7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/HdfsVolumeId.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs;
-
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.util.StringUtils;
-
-import com.google.common.base.Preconditions;
-
-/**
- * HDFS-specific volume identifier which implements {@link VolumeId}. Can be
- * used to differentiate between the data directories on a single datanode. This
- * identifier is only unique on a per-datanode basis.
- */
-@InterfaceStability.Unstable
-@InterfaceAudience.Public
-public class HdfsVolumeId implements VolumeId {
-
- private final byte[] id;
-
- public HdfsVolumeId(byte[] id) {
- Preconditions.checkNotNull(id, "id cannot be null");
- this.id = id;
- }
-
- @Override
- public int compareTo(VolumeId arg0) {
- if (arg0 == null) {
- return 1;
- }
- return hashCode() - arg0.hashCode();
- }
-
- @Override
- public int hashCode() {
- return new HashCodeBuilder().append(id).toHashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == null || obj.getClass() != getClass()) {
- return false;
- }
- if (obj == this) {
- return true;
- }
- HdfsVolumeId that = (HdfsVolumeId) obj;
- return new EqualsBuilder().append(this.id, that.id).isEquals();
- }
-
- @Override
- public String toString() {
- return StringUtils.byteToHexString(id);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eee4d716/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/VolumeId.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/VolumeId.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/VolumeId.java
deleted file mode 100644
index e56e304..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/VolumeId.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * Opaque interface that identifies a disk location. Subclasses
- * should implement {@link Comparable} and override both equals and hashCode.
- */
-@InterfaceStability.Unstable
-@InterfaceAudience.Public
-public interface VolumeId extends Comparable<VolumeId> {
-
- @Override
- abstract public int compareTo(VolumeId arg0);
-
- @Override
- abstract public int hashCode();
-
- @Override
- abstract public boolean equals(Object obj);
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eee4d716/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockStorageLocationUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockStorageLocationUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockStorageLocationUtil.java
deleted file mode 100644
index a1cd555..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockStorageLocationUtil.java
+++ /dev/null
@@ -1,368 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CancellationException;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.BlockStorageLocation;
-import org.apache.hadoop.fs.HdfsVolumeId;
-import org.apache.hadoop.fs.VolumeId;
-import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
-import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
-import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.security.token.Token;
-import org.apache.htrace.Span;
-import org.apache.htrace.Trace;
-import org.apache.htrace.TraceScope;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-class BlockStorageLocationUtil {
-
- static final Log LOG = LogFactory
- .getLog(BlockStorageLocationUtil.class);
-
- /**
- * Create a list of {@link VolumeBlockLocationCallable} corresponding to a set
- * of datanodes and blocks. The blocks must all correspond to the same
- * block pool.
- *
- * @param datanodeBlocks
- * Map of datanodes to block replicas at each datanode
- * @return callables Used to query each datanode for location information on
- * the block replicas at the datanode
- */
- private static List<VolumeBlockLocationCallable> createVolumeBlockLocationCallables(
- Configuration conf, Map<DatanodeInfo, List<LocatedBlock>> datanodeBlocks,
- int timeout, boolean connectToDnViaHostname, Span parent) {
-
- if (datanodeBlocks.isEmpty()) {
- return Lists.newArrayList();
- }
-
- // Construct the callables, one per datanode
- List<VolumeBlockLocationCallable> callables =
- new ArrayList<VolumeBlockLocationCallable>();
- for (Map.Entry<DatanodeInfo, List<LocatedBlock>> entry : datanodeBlocks
- .entrySet()) {
- // Construct RPC parameters
- DatanodeInfo datanode = entry.getKey();
- List<LocatedBlock> locatedBlocks = entry.getValue();
- if (locatedBlocks.isEmpty()) {
- continue;
- }
-
- // Ensure that the blocks all are from the same block pool.
- String poolId = locatedBlocks.get(0).getBlock().getBlockPoolId();
- for (LocatedBlock lb : locatedBlocks) {
- if (!poolId.equals(lb.getBlock().getBlockPoolId())) {
- throw new IllegalArgumentException(
- "All blocks to be queried must be in the same block pool: " +
- locatedBlocks.get(0).getBlock() + " and " + lb +
- " are from different pools.");
- }
- }
-
- long[] blockIds = new long[locatedBlocks.size()];
- int i = 0;
- List<Token<BlockTokenIdentifier>> dnTokens =
- new ArrayList<Token<BlockTokenIdentifier>>(
- locatedBlocks.size());
- for (LocatedBlock b : locatedBlocks) {
- blockIds[i++] = b.getBlock().getBlockId();
- dnTokens.add(b.getBlockToken());
- }
- VolumeBlockLocationCallable callable = new VolumeBlockLocationCallable(
- conf, datanode, poolId, blockIds, dnTokens, timeout,
- connectToDnViaHostname, parent);
- callables.add(callable);
- }
- return callables;
- }
-
- /**
- * Queries datanodes for the blocks specified in <code>datanodeBlocks</code>,
- * making one RPC to each datanode. These RPCs are made in parallel using a
- * threadpool.
- *
- * @param datanodeBlocks
- * Map of datanodes to the blocks present on the DN
- * @return metadatas Map of datanodes to block metadata of the DN
- * @throws InvalidBlockTokenException
- * if client does not have read access on a requested block
- */
- static Map<DatanodeInfo, HdfsBlocksMetadata> queryDatanodesForHdfsBlocksMetadata(
- Configuration conf, Map<DatanodeInfo, List<LocatedBlock>> datanodeBlocks,
- int poolsize, int timeoutMs, boolean connectToDnViaHostname)
- throws InvalidBlockTokenException {
-
- List<VolumeBlockLocationCallable> callables =
- createVolumeBlockLocationCallables(conf, datanodeBlocks, timeoutMs,
- connectToDnViaHostname, Trace.currentSpan());
-
- // Use a thread pool to execute the Callables in parallel
- List<Future<HdfsBlocksMetadata>> futures =
- new ArrayList<Future<HdfsBlocksMetadata>>();
- ExecutorService executor = new ScheduledThreadPoolExecutor(poolsize);
- try {
- futures = executor.invokeAll(callables, timeoutMs,
- TimeUnit.MILLISECONDS);
- } catch (InterruptedException e) {
- // Swallow the exception here, because we can return partial results
- }
- executor.shutdown();
-
- Map<DatanodeInfo, HdfsBlocksMetadata> metadatas =
- Maps.newHashMapWithExpectedSize(datanodeBlocks.size());
- // Fill in metadatas with results from DN RPCs, where possible
- for (int i = 0; i < futures.size(); i++) {
- VolumeBlockLocationCallable callable = callables.get(i);
- DatanodeInfo datanode = callable.getDatanodeInfo();
- Future<HdfsBlocksMetadata> future = futures.get(i);
- try {
- HdfsBlocksMetadata metadata = future.get();
- metadatas.put(callable.getDatanodeInfo(), metadata);
- } catch (CancellationException e) {
- LOG.info("Cancelled while waiting for datanode "
- + datanode.getIpcAddr(false) + ": " + e.toString());
- } catch (ExecutionException e) {
- Throwable t = e.getCause();
- if (t instanceof InvalidBlockTokenException) {
- LOG.warn("Invalid access token when trying to retrieve "
- + "information from datanode " + datanode.getIpcAddr(false));
- throw (InvalidBlockTokenException) t;
- }
- else if (t instanceof UnsupportedOperationException) {
- LOG.info("Datanode " + datanode.getIpcAddr(false) + " does not support"
- + " required #getHdfsBlocksMetadata() API");
- throw (UnsupportedOperationException) t;
- } else {
- LOG.info("Failed to query block locations on datanode " +
- datanode.getIpcAddr(false) + ": " + t);
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Could not fetch information from datanode", t);
- }
- } catch (InterruptedException e) {
- // Shouldn't happen, because invokeAll waits for all Futures to be ready
- LOG.info("Interrupted while fetching HdfsBlocksMetadata");
- }
- }
-
- return metadatas;
- }
-
- /**
- * Group the per-replica {@link VolumeId} info returned from
- * {@link DFSClient#queryDatanodesForHdfsBlocksMetadata(Map)} to be
- * associated
- * with the corresponding {@link LocatedBlock}.
- *
- * @param blocks
- * Original LocatedBlock array
- * @param metadatas
- * VolumeId information for the replicas on each datanode
- * @return blockVolumeIds per-replica VolumeId information associated with the
- * parent LocatedBlock
- */
- static Map<LocatedBlock, List<VolumeId>> associateVolumeIdsWithBlocks(
- List<LocatedBlock> blocks,
- Map<DatanodeInfo, HdfsBlocksMetadata> metadatas) {
-
- // Initialize mapping of ExtendedBlock to LocatedBlock.
- // Used to associate results from DN RPCs to the parent LocatedBlock
- Map<Long, LocatedBlock> blockIdToLocBlock =
- new HashMap<Long, LocatedBlock>();
- for (LocatedBlock b : blocks) {
- blockIdToLocBlock.put(b.getBlock().getBlockId(), b);
- }
-
- // Initialize the mapping of blocks -> list of VolumeIds, one per replica
- // This is filled out with real values from the DN RPCs
- Map<LocatedBlock, List<VolumeId>> blockVolumeIds =
- new HashMap<LocatedBlock, List<VolumeId>>();
- for (LocatedBlock b : blocks) {
- ArrayList<VolumeId> l = new ArrayList<VolumeId>(b.getLocations().length);
- for (int i = 0; i < b.getLocations().length; i++) {
- l.add(null);
- }
- blockVolumeIds.put(b, l);
- }
-
- // Iterate through the list of metadatas (one per datanode).
- // For each metadata, if it's valid, insert its volume location information
- // into the Map returned to the caller
- for (Map.Entry<DatanodeInfo, HdfsBlocksMetadata> entry : metadatas.entrySet()) {
- DatanodeInfo datanode = entry.getKey();
- HdfsBlocksMetadata metadata = entry.getValue();
- // Check if metadata is valid
- if (metadata == null) {
- continue;
- }
- long[] metaBlockIds = metadata.getBlockIds();
- List<byte[]> metaVolumeIds = metadata.getVolumeIds();
- List<Integer> metaVolumeIndexes = metadata.getVolumeIndexes();
- // Add VolumeId for each replica in the HdfsBlocksMetadata
- for (int j = 0; j < metaBlockIds.length; j++) {
- int volumeIndex = metaVolumeIndexes.get(j);
- long blockId = metaBlockIds[j];
- // Skip if block wasn't found, or not a valid index into metaVolumeIds
- // Also skip if the DN responded with a block we didn't ask for
- if (volumeIndex == Integer.MAX_VALUE
- || volumeIndex >= metaVolumeIds.size()
- || !blockIdToLocBlock.containsKey(blockId)) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("No data for block " + blockId);
- }
- continue;
- }
- // Get the VolumeId by indexing into the list of VolumeIds
- // provided by the datanode
- byte[] volumeId = metaVolumeIds.get(volumeIndex);
- HdfsVolumeId id = new HdfsVolumeId(volumeId);
- // Find out which index we are in the LocatedBlock's replicas
- LocatedBlock locBlock = blockIdToLocBlock.get(blockId);
- DatanodeInfo[] dnInfos = locBlock.getLocations();
- int index = -1;
- for (int k = 0; k < dnInfos.length; k++) {
- if (dnInfos[k].equals(datanode)) {
- index = k;
- break;
- }
- }
- if (index < 0) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Datanode responded with a block volume id we did" +
- " not request, omitting.");
- }
- continue;
- }
- // Place VolumeId at the same index as the DN's index in the list of
- // replicas
- List<VolumeId> volumeIds = blockVolumeIds.get(locBlock);
- volumeIds.set(index, id);
- }
- }
- return blockVolumeIds;
- }
-
- /**
- * Helper method to combine a list of {@link LocatedBlock} with associated
- * {@link VolumeId} information to form a list of {@link BlockStorageLocation}
- * .
- */
- static BlockStorageLocation[] convertToVolumeBlockLocations(
- List<LocatedBlock> blocks,
- Map<LocatedBlock, List<VolumeId>> blockVolumeIds) throws IOException {
- // Construct the final return value of VolumeBlockLocation[]
- BlockLocation[] locations = DFSUtilClient.locatedBlocks2Locations(blocks);
- List<BlockStorageLocation> volumeBlockLocs =
- new ArrayList<BlockStorageLocation>(locations.length);
- for (int i = 0; i < locations.length; i++) {
- LocatedBlock locBlock = blocks.get(i);
- List<VolumeId> volumeIds = blockVolumeIds.get(locBlock);
- BlockStorageLocation bsLoc = new BlockStorageLocation(locations[i],
- volumeIds.toArray(new VolumeId[0]));
- volumeBlockLocs.add(bsLoc);
- }
- return volumeBlockLocs.toArray(new BlockStorageLocation[] {});
- }
-
- /**
- * Callable that sets up an RPC proxy to a datanode and queries it for
- * volume location information for a list of ExtendedBlocks.
- */
- private static class VolumeBlockLocationCallable implements
- Callable<HdfsBlocksMetadata> {
-
- private final Configuration configuration;
- private final int timeout;
- private final DatanodeInfo datanode;
- private final String poolId;
- private final long[] blockIds;
- private final List<Token<BlockTokenIdentifier>> dnTokens;
- private final boolean connectToDnViaHostname;
- private final Span parentSpan;
-
- VolumeBlockLocationCallable(Configuration configuration,
- DatanodeInfo datanode, String poolId, long []blockIds,
- List<Token<BlockTokenIdentifier>> dnTokens, int timeout,
- boolean connectToDnViaHostname, Span parentSpan) {
- this.configuration = configuration;
- this.timeout = timeout;
- this.datanode = datanode;
- this.poolId = poolId;
- this.blockIds = blockIds;
- this.dnTokens = dnTokens;
- this.connectToDnViaHostname = connectToDnViaHostname;
- this.parentSpan = parentSpan;
- }
-
- public DatanodeInfo getDatanodeInfo() {
- return datanode;
- }
-
- @Override
- public HdfsBlocksMetadata call() throws Exception {
- HdfsBlocksMetadata metadata = null;
- // Create the RPC proxy and make the RPC
- ClientDatanodeProtocol cdp = null;
- TraceScope scope =
- Trace.startSpan("getHdfsBlocksMetadata", parentSpan);
- try {
- cdp = DFSUtil.createClientDatanodeProtocolProxy(datanode, configuration,
- timeout, connectToDnViaHostname);
- metadata = cdp.getHdfsBlocksMetadata(poolId, blockIds, dnTokens);
- } catch (IOException e) {
- // Bubble this up to the caller, handle with the Future
- throw e;
- } finally {
- scope.close();
- if (cdp != null) {
- RPC.stopProxy(cdp);
- }
- }
- return metadata;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eee4d716/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 3f4621e..e1c8a8a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -42,7 +42,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
@@ -70,7 +69,6 @@ import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.BlockStorageLocation;
import org.apache.hadoop.fs.CacheFlag;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
@@ -91,7 +89,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.fs.UnresolvedLinkException;
-import org.apache.hadoop.fs.VolumeId;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.AclEntry;
@@ -121,7 +118,6 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.EncryptionZoneIterator;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
@@ -924,87 +920,6 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
}
/**
- * Get block location information about a list of {@link HdfsBlockLocation}.
- * Used by {@link DistributedFileSystem#getFileBlockStorageLocations(List)} to
- * get {@link BlockStorageLocation}s for blocks returned by
- * {@link DistributedFileSystem#getFileBlockLocations(org.apache.hadoop.fs.FileStatus, long, long)}
- * .
- *
- * This is done by making a round of RPCs to the associated datanodes, asking
- * the volume of each block replica. The returned array of
- * {@link BlockStorageLocation} expose this information as a
- * {@link VolumeId}.
- *
- * @param blockLocations
- * target blocks on which to query volume location information
- * @return volumeBlockLocations original block array augmented with additional
- * volume location information for each replica.
- */
- public BlockStorageLocation[] getBlockStorageLocations(
- List<BlockLocation> blockLocations) throws IOException,
- UnsupportedOperationException, InvalidBlockTokenException {
- checkOpen();
- if (!getConf().isHdfsBlocksMetadataEnabled()) {
- throw new UnsupportedOperationException("Datanode-side support for " +
- "getVolumeBlockLocations() must also be enabled in the client " +
- "configuration.");
- }
- // Downcast blockLocations and fetch out required LocatedBlock(s)
- List<LocatedBlock> blocks = new ArrayList<LocatedBlock>();
- for (BlockLocation loc : blockLocations) {
- if (!(loc instanceof HdfsBlockLocation)) {
- throw new ClassCastException("DFSClient#getVolumeBlockLocations " +
- "expected to be passed HdfsBlockLocations");
- }
- HdfsBlockLocation hdfsLoc = (HdfsBlockLocation) loc;
- blocks.add(hdfsLoc.getLocatedBlock());
- }
-
- // Re-group the LocatedBlocks to be grouped by datanodes, with the values
- // a list of the LocatedBlocks on the datanode.
- Map<DatanodeInfo, List<LocatedBlock>> datanodeBlocks =
- new LinkedHashMap<DatanodeInfo, List<LocatedBlock>>();
- for (LocatedBlock b : blocks) {
- for (DatanodeInfo info : b.getLocations()) {
- if (!datanodeBlocks.containsKey(info)) {
- datanodeBlocks.put(info, new ArrayList<LocatedBlock>());
- }
- List<LocatedBlock> l = datanodeBlocks.get(info);
- l.add(b);
- }
- }
-
- // Make RPCs to the datanodes to get volume locations for its replicas
- TraceScope scope =
- Trace.startSpan("getBlockStorageLocations", traceSampler);
- Map<DatanodeInfo, HdfsBlocksMetadata> metadatas;
- try {
- metadatas = BlockStorageLocationUtil.
- queryDatanodesForHdfsBlocksMetadata(conf, datanodeBlocks,
- getConf().getFileBlockStorageLocationsNumThreads(),
- getConf().getFileBlockStorageLocationsTimeoutMs(),
- getConf().isConnectToDnViaHostname());
- if (LOG.isTraceEnabled()) {
- LOG.trace("metadata returned: "
- + Joiner.on("\n").withKeyValueSeparator("=").join(metadatas));
- }
- } finally {
- scope.close();
- }
-
- // Regroup the returned VolumeId metadata to again be grouped by
- // LocatedBlock rather than by datanode
- Map<LocatedBlock, List<VolumeId>> blockVolumeIds = BlockStorageLocationUtil
- .associateVolumeIdsWithBlocks(blocks, metadatas);
-
- // Combine original BlockLocations with new VolumeId information
- BlockStorageLocation[] volumeBlockLocations = BlockStorageLocationUtil
- .convertToVolumeBlockLocations(blocks, blockVolumeIds);
-
- return volumeBlockLocations;
- }
-
- /**
* Decrypts a EDEK by consulting the KeyProvider.
*/
private KeyVersion decryptEncryptedDataEncryptionKey(FileEncryptionInfo
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eee4d716/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index c9425ed..f3fc037 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -50,8 +50,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_USER_HOME_DIR_PREFIX_DEFAULT = "/user";
public static final String DFS_CHECKSUM_TYPE_KEY = "dfs.checksum.type";
public static final String DFS_CHECKSUM_TYPE_DEFAULT = "CRC32C";
- public static final String DFS_HDFS_BLOCKS_METADATA_ENABLED = "dfs.datanode.hdfs-blocks-metadata.enabled";
- public static final boolean DFS_HDFS_BLOCKS_METADATA_ENABLED_DEFAULT = false;
public static final String DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT =
HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT;
@@ -985,10 +983,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_CLIENT_CONTEXT = "dfs.client.context";
public static final String DFS_CLIENT_CONTEXT_DEFAULT = "default";
- public static final String DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_NUM_THREADS = "dfs.client.file-block-storage-locations.num-threads";
- public static final int DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_NUM_THREADS_DEFAULT = 10;
- public static final String DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_TIMEOUT_MS = "dfs.client.file-block-storage-locations.timeout.millis";
- public static final int DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_TIMEOUT_MS_DEFAULT = 1000;
public static final String DFS_CLIENT_DATANODE_RESTART_TIMEOUT_KEY = "dfs.client.datanode-restart.timeout";
public static final long DFS_CLIENT_DATANODE_RESTART_TIMEOUT_DEFAULT = 30;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eee4d716/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 0197cfb..ebed3c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.BlockStorageLocation;
import org.apache.hadoop.fs.BlockStoragePolicySpi;
import org.apache.hadoop.fs.CacheFlag;
import org.apache.hadoop.fs.ContentSummary;
@@ -58,7 +57,6 @@ import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.apache.hadoop.fs.VolumeId;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -85,7 +83,6 @@ import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
-import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.io.Text;
@@ -233,42 +230,6 @@ public class DistributedFileSystem extends FileSystem {
}.resolve(this, absF);
}
- /**
- * This API has been deprecated since the NameNode now tracks datanode
- * storages separately. Storage IDs can be gotten from {@link
- * BlockLocation#getStorageIds()}, which are functionally equivalent to
- * the volume IDs returned here (although a String rather than a byte[]).
- *
- * Used to query storage location information for a list of blocks. This list
- * of blocks is normally constructed via a series of calls to
- * {@link DistributedFileSystem#getFileBlockLocations(Path, long, long)} to
- * get the blocks for ranges of a file.
- *
- * The returned array of {@link BlockStorageLocation} augments
- * {@link BlockLocation} with a {@link VolumeId} per block replica. The
- * VolumeId specifies the volume on the datanode on which the replica resides.
- * The VolumeId associated with a replica may be null because volume
- * information can be unavailable if the corresponding datanode is down or
- * if the requested block is not found.
- *
- * This API is unstable, and datanode-side support is disabled by default. It
- * can be enabled by setting "dfs.datanode.hdfs-blocks-metadata.enabled" to
- * true.
- *
- * @param blocks
- * List of target BlockLocations to query volume location information
- * @return volumeBlockLocations Augmented array of
- * {@link BlockStorageLocation}s containing additional volume location
- * information for each replica of each block.
- */
- @InterfaceStability.Unstable
- @Deprecated
- public BlockStorageLocation[] getFileBlockStorageLocations(
- List<BlockLocation> blocks) throws IOException,
- UnsupportedOperationException, InvalidBlockTokenException {
- return dfs.getBlockStorageLocations(blocks);
- }
-
@Override
public void setVerifyChecksum(boolean verifyChecksum) {
this.verifyChecksum = verifyChecksum;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eee4d716/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java
index cdb9371..99e0e8e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java
@@ -138,8 +138,6 @@ public class HdfsConfiguration extends Configuration {
DFSConfigKeys.DFS_NAMESERVICES),
new DeprecationDelta("dfs.federation.nameservice.id",
DFSConfigKeys.DFS_NAMESERVICE_ID),
- new DeprecationDelta("dfs.client.file-block-storage-locations.timeout",
- DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_TIMEOUT_MS),
});
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eee4d716/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
index a257e32..d014a1f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
@@ -88,9 +88,6 @@ public class DfsClientConf {
private final String taskId;
private final FsPermission uMask;
private final boolean connectToDnViaHostname;
- private final boolean hdfsBlocksMetadataEnabled;
- private final int fileBlockStorageLocationsNumThreads;
- private final int fileBlockStorageLocationsTimeoutMs;
private final int retryTimesForGetLastBlockLength;
private final int retryIntervalForGetLastBlockLength;
private final long datanodeRestartTimeout;
@@ -190,15 +187,6 @@ public class DfsClientConf {
uMask = FsPermission.getUMask(conf);
connectToDnViaHostname = conf.getBoolean(DFS_CLIENT_USE_DN_HOSTNAME,
DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT);
- hdfsBlocksMetadataEnabled = conf.getBoolean(
- DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED,
- DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED_DEFAULT);
- fileBlockStorageLocationsNumThreads = conf.getInt(
- DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_NUM_THREADS,
- DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_NUM_THREADS_DEFAULT);
- fileBlockStorageLocationsTimeoutMs = conf.getInt(
- DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_TIMEOUT_MS,
- DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_TIMEOUT_MS_DEFAULT);
datanodeRestartTimeout = conf.getLong(
DFS_CLIENT_DATANODE_RESTART_TIMEOUT_KEY,
@@ -429,27 +417,6 @@ public class DfsClientConf {
}
/**
- * @return the hdfsBlocksMetadataEnabled
- */
- public boolean isHdfsBlocksMetadataEnabled() {
- return hdfsBlocksMetadataEnabled;
- }
-
- /**
- * @return the fileBlockStorageLocationsNumThreads
- */
- public int getFileBlockStorageLocationsNumThreads() {
- return fileBlockStorageLocationsNumThreads;
- }
-
- /**
- * @return the getFileBlockStorageLocationsTimeoutMs
- */
- public int getFileBlockStorageLocationsTimeoutMs() {
- return fileBlockStorageLocationsTimeoutMs;
- }
-
- /**
* @return the retryTimesForGetLastBlockLength
*/
public int getRetryTimesForGetLastBlockLength() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eee4d716/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
index 75f8907..da8f4ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
@@ -111,25 +111,6 @@ public interface ClientDatanodeProtocol {
Token<BlockTokenIdentifier> token) throws IOException;
/**
- * Retrieves volume location information about a list of blocks on a datanode.
- * This is in the form of an opaque {@link org.apache.hadoop.fs.VolumeId}
- * for each configured data directory, which is not guaranteed to be
- * the same across DN restarts.
- *
- * @param blockPoolId the pool to query
- * @param blockIds
- * list of blocks on the local datanode
- * @param tokens
- * block access tokens corresponding to the requested blocks
- * @return an HdfsBlocksMetadata that associates {@link ExtendedBlock}s with
- * data directories
- * @throws IOException
- * if datanode is unreachable, or replica is not found on datanode
- */
- HdfsBlocksMetadata getHdfsBlocksMetadata(String blockPoolId,
- long []blockIds, List<Token<BlockTokenIdentifier>> tokens) throws IOException;
-
- /**
* Shuts down a datanode.
*
* @param forUpgrade If true, data node does extra prep work before shutting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eee4d716/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsBlocksMetadata.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsBlocksMetadata.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsBlocksMetadata.java
deleted file mode 100644
index 487bbcb..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsBlocksMetadata.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.protocol;
-
-import java.util.List;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-import com.google.common.primitives.Longs;
-
-/**
- * Augments an array of blocks on a datanode with additional information about
- * where the block is stored.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class HdfsBlocksMetadata {
-
- /** The block pool that was queried */
- private final String blockPoolId;
-
- /**
- * List of blocks
- */
- private final long[] blockIds;
-
- /**
- * List of volumes
- */
- private final List<byte[]> volumeIds;
-
- /**
- * List of indexes into <code>volumeIds</code>, one per block in
- * <code>blocks</code>. A value of Integer.MAX_VALUE indicates that the
- * block was not found.
- */
- private final List<Integer> volumeIndexes;
-
- /**
- * Constructs HdfsBlocksMetadata.
- *
- * @param blockIds
- * List of blocks described
- * @param volumeIds
- * List of potential volume identifiers, specifying volumes where
- * blocks may be stored
- * @param volumeIndexes
- * Indexes into the list of volume identifiers, one per block
- */
- public HdfsBlocksMetadata(String blockPoolId,
- long[] blockIds, List<byte[]> volumeIds,
- List<Integer> volumeIndexes) {
- Preconditions.checkArgument(blockIds.length == volumeIndexes.size(),
- "Argument lengths should match");
- this.blockPoolId = blockPoolId;
- this.blockIds = blockIds;
- this.volumeIds = volumeIds;
- this.volumeIndexes = volumeIndexes;
- }
-
- /**
- * Get the array of blocks.
- *
- * @return array of blocks
- */
- public long[] getBlockIds() {
- return blockIds;
- }
-
- /**
- * Get the list of volume identifiers in raw byte form.
- *
- * @return list of ids
- */
- public List<byte[]> getVolumeIds() {
- return volumeIds;
- }
-
- /**
- * Get a list of indexes into the array of {@link VolumeId}s, one per block.
- *
- * @return list of indexes
- */
- public List<Integer> getVolumeIndexes() {
- return volumeIndexes;
- }
-
- @Override
- public String toString() {
- return "Metadata for " + blockIds.length + " blocks in " +
- blockPoolId + ": " + Joiner.on(",").join(Longs.asList(blockIds));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eee4d716/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java
index 2bc6a18..c62d9ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.conf.ReconfigurationUtil.PropertyChange;
import org.apache.hadoop.hdfs.client.BlockReportOptions;
import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
-import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.DeleteBlockPoolRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.DeleteBlockPoolResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlockLocalPathInfoRequestProto;
@@ -39,9 +38,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetDat
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetReconfigurationStatusConfigChangeProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetReconfigurationStatusRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetReconfigurationStatusResponseProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsRequestProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsResponseProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsResponseProto.Builder;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetReplicaVisibleLengthRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetReplicaVisibleLengthResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.ListReconfigurablePropertiesRequestProto;
@@ -54,12 +50,7 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.StartR
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.StartReconfigurationResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.TriggerBlockReportRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.TriggerBlockReportResponseProto;
-import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
-import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
-import org.apache.hadoop.security.token.Token;
-import com.google.common.primitives.Longs;
-import com.google.protobuf.ByteString;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
@@ -143,38 +134,6 @@ public class ClientDatanodeProtocolServerSideTranslatorPB implements
}
@Override
- public GetHdfsBlockLocationsResponseProto getHdfsBlockLocations(
- RpcController controller, GetHdfsBlockLocationsRequestProto request)
- throws ServiceException {
- HdfsBlocksMetadata resp;
- try {
- String poolId = request.getBlockPoolId();
-
- List<Token<BlockTokenIdentifier>> tokens =
- new ArrayList<Token<BlockTokenIdentifier>>(request.getTokensCount());
- for (TokenProto b : request.getTokensList()) {
- tokens.add(PBHelper.convert(b));
- }
- long[] blockIds = Longs.toArray(request.getBlockIdsList());
-
- // Call the real implementation
- resp = impl.getHdfsBlocksMetadata(poolId, blockIds, tokens);
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- List<ByteString> volumeIdsByteStrings =
- new ArrayList<ByteString>(resp.getVolumeIds().size());
- for (byte[] b : resp.getVolumeIds()) {
- volumeIdsByteStrings.add(ByteString.copyFrom(b));
- }
- // Build and return the response
- Builder builder = GetHdfsBlockLocationsResponseProto.newBuilder();
- builder.addAllVolumeIds(volumeIdsByteStrings);
- builder.addAllVolumeIndexes(resp.getVolumeIndexes());
- return builder.build();
- }
-
- @Override
public ShutdownDatanodeResponseProto shutdownDatanode(
RpcController unused, ShutdownDatanodeRequestProto request)
throws ServiceException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eee4d716/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java
index 9d6375b..2146063 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.protocolPB;
import java.io.Closeable;
import java.io.IOException;
import java.net.InetSocketAddress;
-import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -42,15 +41,12 @@ import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeLocalInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.DeleteBlockPoolRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlockLocalPathInfoRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlockLocalPathInfoResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetDatanodeInfoRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetDatanodeInfoResponseProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsRequestProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetReplicaVisibleLengthRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.ListReconfigurablePropertiesRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.ListReconfigurablePropertiesResponseProto;
@@ -70,11 +66,8 @@ import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RpcClientUtil;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.security.token.Token;
-import com.google.common.primitives.Longs;
-import com.google.protobuf.ByteString;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
@@ -243,42 +236,6 @@ public class ClientDatanodeProtocolTranslatorPB implements
}
@Override
- public HdfsBlocksMetadata getHdfsBlocksMetadata(String blockPoolId,
- long[] blockIds,
- List<Token<BlockTokenIdentifier>> tokens) throws IOException {
- List<TokenProto> tokensProtos =
- new ArrayList<TokenProto>(tokens.size());
- for (Token<BlockTokenIdentifier> t : tokens) {
- tokensProtos.add(PBHelper.convert(t));
- }
- // Build the request
- GetHdfsBlockLocationsRequestProto request =
- GetHdfsBlockLocationsRequestProto.newBuilder()
- .setBlockPoolId(blockPoolId)
- .addAllBlockIds(Longs.asList(blockIds))
- .addAllTokens(tokensProtos)
- .build();
- // Send the RPC
- GetHdfsBlockLocationsResponseProto response;
- try {
- response = rpcProxy.getHdfsBlockLocations(NULL_CONTROLLER, request);
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- // List of volumes in the response
- List<ByteString> volumeIdsByteStrings = response.getVolumeIdsList();
- List<byte[]> volumeIds = new ArrayList<byte[]>(volumeIdsByteStrings.size());
- for (ByteString bs : volumeIdsByteStrings) {
- volumeIds.add(bs.toByteArray());
- }
- // Array of indexes into the list of volumes, one per block
- List<Integer> volumeIndexes = response.getVolumeIndexesList();
- // Parsed HdfsVolumeId values, one per block
- return new HdfsBlocksMetadata(blockPoolId, blockIds,
- volumeIds, volumeIndexes);
- }
-
- @Override
public void shutdownDatanode(boolean forUpgrade) throws IOException {
ShutdownDatanodeRequestProto request = ShutdownDatanodeRequestProto
.newBuilder().setForUpgrade(forUpgrade).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eee4d716/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index e265dad..7757d8b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -115,7 +115,6 @@ import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DatanodeLocalInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException;
import org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage;
@@ -346,7 +345,6 @@ public class DataNode extends ReconfigurableBase
ReadaheadPool readaheadPool;
SaslDataTransferClient saslClient;
SaslDataTransferServer saslServer;
- private final boolean getHdfsBlockLocationsEnabled;
private ObjectName dataNodeInfoBeanName;
private Thread checkDiskErrorThread = null;
protected final int checkDiskErrorInterval = 5*1000;
@@ -374,7 +372,6 @@ public class DataNode extends ReconfigurableBase
this.confVersion = null;
this.usersWithLocalPathAccess = null;
this.connectToDnViaHostname = false;
- this.getHdfsBlockLocationsEnabled = false;
this.blockScanner = new BlockScanner(this, conf);
this.pipelineSupportECN = false;
}
@@ -397,9 +394,6 @@ public class DataNode extends ReconfigurableBase
this.connectToDnViaHostname = conf.getBoolean(
DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME,
DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT);
- this.getHdfsBlockLocationsEnabled = conf.getBoolean(
- DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED,
- DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED_DEFAULT);
this.supergroup = conf.get(DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_KEY,
DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT);
this.isPermissionEnabled = conf.getBoolean(
@@ -1624,29 +1618,6 @@ public class DataNode extends ReconfigurableBase
return fis;
}
- @Override
- public HdfsBlocksMetadata getHdfsBlocksMetadata(
- String bpId, long[] blockIds,
- List<Token<BlockTokenIdentifier>> tokens) throws IOException,
- UnsupportedOperationException {
- if (!getHdfsBlockLocationsEnabled) {
- throw new UnsupportedOperationException("Datanode#getHdfsBlocksMetadata "
- + " is not enabled in datanode config");
- }
- if (blockIds.length != tokens.size()) {
- throw new IOException("Differing number of blocks and tokens");
- }
- // Check access for each block
- for (int i = 0; i < blockIds.length; i++) {
- checkBlockToken(new ExtendedBlock(bpId, blockIds[i]),
- tokens.get(i), BlockTokenIdentifier.AccessMode.READ);
- }
-
- DataNodeFaultInjector.get().getHdfsBlocksMetadata();
-
- return data.getHdfsBlocksMetadata(bpId, blockIds);
- }
-
private void checkBlockToken(ExtendedBlock block, Token<BlockTokenIdentifier> token,
AccessMode accessMode) throws IOException {
if (isBlockTokenEnabled) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eee4d716/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
index af6a532..63f0377 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
@@ -40,7 +40,6 @@ import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataStorage;
@@ -561,18 +560,6 @@ public interface FsDatasetSpi<V extends FsVolumeSpi> extends FSDatasetMBean {
) throws IOException;
/**
- * Get a {@link HdfsBlocksMetadata} corresponding to the list of blocks in
- * <code>blocks</code>.
- *
- * @param bpid pool to query
- * @param blockIds List of block ids for which to return metadata
- * @return metadata Metadata for the list of blocks
- * @throws IOException
- */
- HdfsBlocksMetadata getHdfsBlocksMetadata(String bpid,
- long[] blockIds) throws IOException;
-
- /**
* Enable 'trash' for the given dataset. When trash is enabled, files are
* moved to a separate trash directory instead of being deleted immediately.
* This can be useful for example during rolling upgrades.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eee4d716/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index a2bb2c0..74d39f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -28,7 +28,6 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;
-import java.nio.ByteBuffer;
import java.nio.channels.ClosedChannelException;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
@@ -60,12 +59,10 @@ import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.ExtendedBlockId;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
@@ -2658,48 +2655,6 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
return info;
}
- @Override // FsDatasetSpi
- public HdfsBlocksMetadata getHdfsBlocksMetadata(String poolId,
- long[] blockIds) throws IOException {
- List<FsVolumeImpl> curVolumes = volumes.getVolumes();
- // List of VolumeIds, one per volume on the datanode
- List<byte[]> blocksVolumeIds = new ArrayList<>(curVolumes.size());
- // List of indexes into the list of VolumeIds, pointing at the VolumeId of
- // the volume that the block is on
- List<Integer> blocksVolumeIndexes = new ArrayList<Integer>(blockIds.length);
- // Initialize the list of VolumeIds simply by enumerating the volumes
- for (int i = 0; i < curVolumes.size(); i++) {
- blocksVolumeIds.add(ByteBuffer.allocate(4).putInt(i).array());
- }
- // Determine the index of the VolumeId of each block's volume, by comparing
- // the block's volume against the enumerated volumes
- for (int i = 0; i < blockIds.length; i++) {
- long blockId = blockIds[i];
- boolean isValid = false;
-
- ReplicaInfo info = volumeMap.get(poolId, blockId);
- int volumeIndex = 0;
- if (info != null) {
- FsVolumeSpi blockVolume = info.getVolume();
- for (FsVolumeImpl volume : curVolumes) {
- // This comparison of references should be safe
- if (blockVolume == volume) {
- isValid = true;
- break;
- }
- volumeIndex++;
- }
- }
- // Indicates that the block is not present, or not found in a data dir
- if (!isValid) {
- volumeIndex = Integer.MAX_VALUE;
- }
- blocksVolumeIndexes.add(volumeIndex);
- }
- return new HdfsBlocksMetadata(poolId, blockIds,
- blocksVolumeIds, blocksVolumeIndexes);
- }
-
@Override
public void enableTrash(String bpid) {
dataStorage.enableTrash(bpid);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eee4d716/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 7f909d6..06a60d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -1698,30 +1698,6 @@
</property>
<property>
- <name>dfs.datanode.hdfs-blocks-metadata.enabled</name>
- <value>false</value>
- <description>
- Boolean which enables backend datanode-side support for the experimental DistributedFileSystem#getFileVBlockStorageLocations API.
- </description>
-</property>
-
-<property>
- <name>dfs.client.file-block-storage-locations.num-threads</name>
- <value>10</value>
- <description>
- Number of threads used for making parallel RPCs in DistributedFileSystem#getFileBlockStorageLocations().
- </description>
-</property>
-
-<property>
- <name>dfs.client.file-block-storage-locations.timeout.millis</name>
- <value>1000</value>
- <description>
- Timeout (in milliseconds) for the parallel RPCs made in DistributedFileSystem#getFileBlockStorageLocations().
- </description>
-</property>
-
-<property>
<name>dfs.journalnode.rpc-address</name>
<value>0.0.0.0:8485</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eee4d716/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestVolumeId.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestVolumeId.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestVolumeId.java
deleted file mode 100644
index 5bc1a7b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestVolumeId.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs;
-
-import org.junit.Test;
-import static org.junit.Assert.*;
-
-public class TestVolumeId {
-
- @Test
- public void testEquality() {
- final VolumeId id1 = new HdfsVolumeId(new byte[] { (byte)0, (byte)0 });
- testEq(true, id1, id1);
-
- final VolumeId id2 = new HdfsVolumeId(new byte[] { (byte)0, (byte)1 });
- testEq(true, id2, id2);
- testEq(false, id1, id2);
-
- final VolumeId id3 = new HdfsVolumeId(new byte[] { (byte)1, (byte)0 });
- testEq(true, id3, id3);
- testEq(false, id1, id3);
-
- // same as 2, but "invalid":
- final VolumeId id2copy1 = new HdfsVolumeId(new byte[] { (byte)0, (byte)1 });
-
- testEq(true, id2, id2copy1);
-
- // same as 2copy1:
- final VolumeId id2copy2 = new HdfsVolumeId(new byte[] { (byte)0, (byte)1 });
-
- testEq(true, id2, id2copy2);
-
- testEqMany(true, new VolumeId[] { id2, id2copy1, id2copy2 });
-
- testEqMany(false, new VolumeId[] { id1, id2, id3 });
- }
-
- @SuppressWarnings("unchecked")
- private <T> void testEq(final boolean eq, Comparable<T> id1, Comparable<T> id2) {
- final int h1 = id1.hashCode();
- final int h2 = id2.hashCode();
-
- // eq reflectivity:
- assertTrue(id1.equals(id1));
- assertTrue(id2.equals(id2));
- assertEquals(0, id1.compareTo((T)id1));
- assertEquals(0, id2.compareTo((T)id2));
-
- // eq symmetry:
- assertEquals(eq, id1.equals(id2));
- assertEquals(eq, id2.equals(id1));
-
- // null comparison:
- assertFalse(id1.equals(null));
- assertFalse(id2.equals(null));
-
- // compareTo:
- assertEquals(eq, 0 == id1.compareTo((T)id2));
- assertEquals(eq, 0 == id2.compareTo((T)id1));
- // compareTo must be antisymmetric:
- assertEquals(sign(id1.compareTo((T)id2)), -sign(id2.compareTo((T)id1)));
-
- // compare with null should never return 0 to be consistent with #equals():
- assertTrue(id1.compareTo(null) != 0);
- assertTrue(id2.compareTo(null) != 0);
-
- // check that hash codes did not change:
- assertEquals(h1, id1.hashCode());
- assertEquals(h2, id2.hashCode());
- if (eq) {
- // in this case the hash codes must be the same:
- assertEquals(h1, h2);
- }
- }
-
- private static int sign(int x) {
- if (x == 0) {
- return 0;
- } else if (x > 0) {
- return 1;
- } else {
- return -1;
- }
- }
-
- @SuppressWarnings("unchecked")
- private <T> void testEqMany(final boolean eq, Comparable<T>... volumeIds) {
- Comparable<T> vidNext;
- int sum = 0;
- for (int i=0; i<volumeIds.length; i++) {
- if (i == volumeIds.length - 1) {
- vidNext = volumeIds[0];
- } else {
- vidNext = volumeIds[i + 1];
- }
- testEq(eq, volumeIds[i], vidNext);
- sum += sign(volumeIds[i].compareTo((T)vidNext));
- }
- // the comparison relationship must always be acyclic:
- assertTrue(sum < volumeIds.length);
- }
-
- /*
- * Test HdfsVolumeId(new byte[0]) instances: show that we permit such
- * objects, they are still valid, and obey the same equality
- * rules other objects do.
- */
- @Test
- public void testIdEmptyBytes() {
- final VolumeId idEmpty1 = new HdfsVolumeId(new byte[0]);
- final VolumeId idEmpty2 = new HdfsVolumeId(new byte[0]);
- final VolumeId idNotEmpty = new HdfsVolumeId(new byte[] { (byte)1 });
-
- testEq(true, idEmpty1, idEmpty2);
- testEq(false, idEmpty1, idNotEmpty);
- testEq(false, idEmpty2, idNotEmpty);
- }
-
- /*
- * test #toString() for typical VolumeId equality classes
- */
- @Test
- public void testToString() {
- String strEmpty = new HdfsVolumeId(new byte[] {}).toString();
- assertNotNull(strEmpty);
-
- String strNotEmpty = new HdfsVolumeId(new byte[] { (byte)1 }).toString();
- assertNotNull(strNotEmpty);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eee4d716/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index 4b76c3f..a85d415 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -20,8 +20,6 @@ package org.apache.hadoop.hdfs;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.eq;
@@ -38,20 +36,16 @@ import java.net.SocketTimeoutException;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
import java.util.Set;
-import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.BlockStorageLocation;
import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -65,21 +59,16 @@ import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.fs.VolumeId;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
import org.apache.hadoop.hdfs.net.Peer;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.datanode.DataNodeFaultInjector;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
-import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.hdfs.web.WebHdfsConstants;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.DataChecksum;
@@ -88,12 +77,6 @@ import org.apache.log4j.Level;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.InOrder;
-import org.mockito.Mockito;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-import com.google.common.base.Supplier;
-import com.google.common.collect.Lists;
public class TestDistributedFileSystem {
private static final Random RAN = new Random();
@@ -217,12 +200,6 @@ public class TestDistributedFileSystem {
GenericTestUtils.assertExceptionContains("Filesystem closed", ioe);
}
try {
- dfsClient.getBlockStorageLocations(new ArrayList<BlockLocation>());
- fail("getBlockStorageLocations using a closed filesystem!");
- } catch (IOException ioe) {
- GenericTestUtils.assertExceptionContains("Filesystem closed", ioe);
- }
- try {
dfsClient.createSymlink("target", "link", true);
fail("createSymlink using a closed filesystem!");
} catch (IOException ioe) {
@@ -886,213 +863,6 @@ public class TestDistributedFileSystem {
}
}
- /**
- * Tests the normal path of batching up BlockLocation[]s to be passed to a
- * single
- * {@link DistributedFileSystem#getFileBlockStorageLocations(java.util.List)}
- * call
- */
- @Test(timeout=60000)
- public void testGetFileBlockStorageLocationsBatching() throws Exception {
- final Configuration conf = getTestConfiguration();
- ((Log4JLogger)ProtobufRpcEngine.LOG).getLogger().setLevel(Level.TRACE);
- ((Log4JLogger)BlockStorageLocationUtil.LOG).getLogger().setLevel(Level.TRACE);
- ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.TRACE);
-
- conf.setBoolean(DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED,
- true);
- final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
- .numDataNodes(2).build();
- try {
- final DistributedFileSystem fs = cluster.getFileSystem();
- // Create two files
- final Path tmpFile1 = new Path("/tmpfile1.dat");
- final Path tmpFile2 = new Path("/tmpfile2.dat");
- DFSTestUtil.createFile(fs, tmpFile1, 1024, (short) 2, 0xDEADDEADl);
- DFSTestUtil.createFile(fs, tmpFile2, 1024, (short) 2, 0xDEADDEADl);
- // Make sure files are fully replicated before continuing
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
- @Override
- public Boolean get() {
- try {
- List<BlockLocation> list = Lists.newArrayList();
- list.addAll(Arrays.asList(fs.getFileBlockLocations(tmpFile1, 0,
- 1024)));
- list.addAll(Arrays.asList(fs.getFileBlockLocations(tmpFile2, 0,
- 1024)));
- int totalRepl = 0;
- for (BlockLocation loc : list) {
- totalRepl += loc.getHosts().length;
- }
- if (totalRepl == 4) {
- return true;
- }
- } catch(IOException e) {
- // swallow
- }
- return false;
- }
- }, 500, 30000);
- // Get locations of blocks of both files and concat together
- BlockLocation[] blockLocs1 = fs.getFileBlockLocations(tmpFile1, 0, 1024);
- BlockLocation[] blockLocs2 = fs.getFileBlockLocations(tmpFile2, 0, 1024);
- BlockLocation[] blockLocs = (BlockLocation[]) ArrayUtils.addAll(blockLocs1,
- blockLocs2);
- // Fetch VolumeBlockLocations in batch
- BlockStorageLocation[] locs = fs.getFileBlockStorageLocations(Arrays
- .asList(blockLocs));
- int counter = 0;
- // Print out the list of ids received for each block
- for (BlockStorageLocation l : locs) {
- for (int i = 0; i < l.getVolumeIds().length; i++) {
- VolumeId id = l.getVolumeIds()[i];
- String name = l.getNames()[i];
- if (id != null) {
- System.out.println("Datanode " + name + " has block " + counter
- + " on volume id " + id.toString());
- }
- }
- counter++;
- }
- assertEquals("Expected two HdfsBlockLocations for two 1-block files", 2,
- locs.length);
- for (BlockStorageLocation l : locs) {
- assertEquals("Expected two replicas for each block", 2,
- l.getVolumeIds().length);
- for (int i = 0; i < l.getVolumeIds().length; i++) {
- VolumeId id = l.getVolumeIds()[i];
- String name = l.getNames()[i];
- assertTrue("Expected block to be valid on datanode " + name,
- id != null);
- }
- }
- } finally {
- cluster.shutdown();
- }
- }
-
- /**
- * Tests error paths for
- * {@link DistributedFileSystem#getFileBlockStorageLocations(java.util.List)}
- */
- @Test(timeout=60000)
- public void testGetFileBlockStorageLocationsError() throws Exception {
- final Configuration conf = getTestConfiguration();
- conf.setBoolean(DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED,
- true);
- conf.setInt(
- DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_TIMEOUT_MS, 1500);
- conf.setInt(
- CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
-
- MiniDFSCluster cluster = null;
- try {
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
- cluster.getDataNodes();
- final DistributedFileSystem fs = cluster.getFileSystem();
-
- // Create a few files and add together their block locations into
- // a list.
- final Path tmpFile1 = new Path("/errorfile1.dat");
- final Path tmpFile2 = new Path("/errorfile2.dat");
-
- DFSTestUtil.createFile(fs, tmpFile1, 1024, (short) 2, 0xDEADDEADl);
- DFSTestUtil.createFile(fs, tmpFile2, 1024, (short) 2, 0xDEADDEADl);
-
- // Make sure files are fully replicated before continuing
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
- @Override
- public Boolean get() {
- try {
- List<BlockLocation> list = Lists.newArrayList();
- list.addAll(Arrays.asList(fs.getFileBlockLocations(tmpFile1, 0,
- 1024)));
- list.addAll(Arrays.asList(fs.getFileBlockLocations(tmpFile2, 0,
- 1024)));
- int totalRepl = 0;
- for (BlockLocation loc : list) {
- totalRepl += loc.getHosts().length;
- }
- if (totalRepl == 4) {
- return true;
- }
- } catch(IOException e) {
- // swallow
- }
- return false;
- }
- }, 500, 30000);
-
- BlockLocation[] blockLocs1 = fs.getFileBlockLocations(tmpFile1, 0, 1024);
- BlockLocation[] blockLocs2 = fs.getFileBlockLocations(tmpFile2, 0, 1024);
-
- List<BlockLocation> allLocs = Lists.newArrayList();
- allLocs.addAll(Arrays.asList(blockLocs1));
- allLocs.addAll(Arrays.asList(blockLocs2));
-
- // Stall on the DN to test the timeout
- DataNodeFaultInjector injector = Mockito.mock(DataNodeFaultInjector.class);
- Mockito.doAnswer(new Answer<Void>() {
- @Override
- public Void answer(InvocationOnMock invocation) throws Throwable {
- Thread.sleep(3000);
- return null;
- }
- }).when(injector).getHdfsBlocksMetadata();
- DataNodeFaultInjector.instance = injector;
-
- BlockStorageLocation[] locs = fs.getFileBlockStorageLocations(allLocs);
- for (BlockStorageLocation loc: locs) {
- assertEquals(
- "Found more than 0 cached hosts although RPCs supposedly timed out",
- 0, loc.getCachedHosts().length);
- }
-
- // Restore a default injector
- DataNodeFaultInjector.instance = new DataNodeFaultInjector();
-
- // Stop a datanode to simulate a failure.
- DataNodeProperties stoppedNode = cluster.stopDataNode(0);
-
- // Fetch VolumeBlockLocations
- locs = fs.getFileBlockStorageLocations(allLocs);
- assertEquals("Expected two HdfsBlockLocation for two 1-block files", 2,
- locs.length);
-
- for (BlockStorageLocation l : locs) {
- assertEquals("Expected two replicas for each block", 2,
- l.getHosts().length);
- assertEquals("Expected two VolumeIDs for each block", 2,
- l.getVolumeIds().length);
- assertTrue("Expected one valid and one invalid volume",
- (l.getVolumeIds()[0] == null) ^ (l.getVolumeIds()[1] == null));
- }
-
- // Start the datanode again, and remove one of the blocks.
- // This is a different type of failure where the block itself
- // is invalid.
- cluster.restartDataNode(stoppedNode, true /*keepPort*/);
- cluster.waitActive();
-
- fs.delete(tmpFile2, true);
- HATestUtil.waitForNNToIssueDeletions(cluster.getNameNode());
- cluster.triggerHeartbeats();
- HATestUtil.waitForDNDeletions(cluster);
-
- locs = fs.getFileBlockStorageLocations(allLocs);
- assertEquals("Expected two HdfsBlockLocations for two 1-block files", 2,
- locs.length);
- assertNotNull(locs[0].getVolumeIds()[0]);
- assertNotNull(locs[0].getVolumeIds()[1]);
- assertNull(locs[1].getVolumeIds()[0]);
- assertNull(locs[1].getVolumeIds()[1]);
- } finally {
- if (cluster != null) {
- cluster.shutdown();
- }
- }
- }
-
@Test
public void testCreateWithCustomChecksum() throws Exception {
Configuration conf = getTestConfiguration();