You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ki...@apache.org on 2020/02/25 17:33:44 UTC
[hadoop] branch branch-2.10 updated: Revert "HDFS-6874. Add
GETFILEBLOCKLOCATIONS operation to HttpFS. Contributed by Weiwei Yang"
This is an automated email from the ASF dual-hosted git repository.
kihwal pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.10 by this push:
new e80b5ec Revert "HDFS-6874. Add GETFILEBLOCKLOCATIONS operation to HttpFS. Contributed by Weiwei Yang"
e80b5ec is described below
commit e80b5ec58dbad292ee4c3604d07e1f6c29603d34
Author: Kihwal Lee <ki...@apache.org>
AuthorDate: Tue Feb 25 11:33:20 2020 -0600
Revert "HDFS-6874. Add GETFILEBLOCKLOCATIONS operation to HttpFS. Contributed by Weiwei Yang"
This reverts commit b4a108fa9f38ee028978474fe6c298bbd88fda7a.
Conflicts:
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
---
.../hadoop/fs/http/client/HttpFSFileSystem.java | 42 ----------
.../apache/hadoop/fs/http/server/FSOperations.java | 38 ---------
.../fs/http/server/HttpFSParametersProvider.java | 3 +-
.../apache/hadoop/fs/http/server/HttpFSServer.java | 21 +----
.../hadoop/fs/http/client/BaseTestHttpFSWith.java | 89 +---------------------
5 files changed, 3 insertions(+), 190 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
index 6e39f5b..596aef7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
@@ -23,12 +23,9 @@ import java.util.Collection;
import java.util.EnumSet;
import java.util.List;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.type.MapType;
import com.google.common.base.Charsets;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.DelegationTokenRenewer;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -124,8 +121,6 @@ public class HttpFSFileSystem extends FileSystem
public static final String NEW_LENGTH_PARAM = "newlength";
public static final String START_AFTER_PARAM = "startAfter";
public static final String POLICY_NAME_PARAM = "storagepolicy";
- public static final String OFFSET_PARAM = "offset";
- public static final String LENGTH_PARAM = "length";
public static final String SNAPSHOT_NAME_PARAM = "snapshotname";
public static final String OLD_SNAPSHOT_NAME_PARAM = "oldsnapshotname";
@@ -218,7 +213,6 @@ public class HttpFSFileSystem extends FileSystem
public static final String STORAGE_POLICIES_JSON = "BlockStoragePolicies";
public static final String STORAGE_POLICY_JSON = "BlockStoragePolicy";
- public static final String BLOCK_LOCATIONS_JSON = "BlockLocations";
public static final int HTTP_TEMPORARY_REDIRECT = 307;
@@ -1429,42 +1423,6 @@ public class HttpFSFileSystem extends FileSystem
return createStoragePolicy((JSONObject) json.get(STORAGE_POLICY_JSON));
}
- @Override
- public BlockLocation[] getFileBlockLocations(FileStatus file, long start,
- long len) throws IOException {
- Map<String, String> params = new HashMap<String, String>();
- params.put(OP_PARAM, Operation.GETFILEBLOCKLOCATIONS.toString());
- params.put(OFFSET_PARAM, Long.toString(start));
- params.put(LENGTH_PARAM, Long.toString(len));
- HttpURLConnection conn =
- getConnection(Operation.GETFILEBLOCKLOCATIONS.getMethod(), params,
- file.getPath(), true);
- HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
- JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
- return toBlockLocations(json);
- }
-
- private BlockLocation[] toBlockLocations(JSONObject json)
- throws IOException {
- ObjectMapper mapper = new ObjectMapper();
- MapType subType = mapper.getTypeFactory().constructMapType(
- Map.class,
- String.class,
- BlockLocation[].class);
- MapType rootType = mapper.getTypeFactory().constructMapType(
- Map.class,
- mapper.constructType(String.class),
- mapper.constructType(subType));
-
- Map<String, Map<String, BlockLocation[]>> jsonMap = mapper
- .readValue(json.toJSONString(), rootType);
- Map<String, BlockLocation[]> locationMap = jsonMap
- .get(BLOCK_LOCATIONS_JSON);
- BlockLocation[] locationArray = locationMap.get(
- BlockLocation.class.getSimpleName());
- return locationArray;
- }
-
private BlockStoragePolicy createStoragePolicy(JSONObject policyJson)
throws IOException {
byte id = ((Number) policyJson.get("id")).byteValue();
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
index d389d0b..5d5f9af 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.fs.http.server;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.BlockStoragePolicySpi;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileChecksum;
@@ -38,7 +37,6 @@ import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.web.JsonUtil;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.lib.service.FileSystemAccess;
import org.apache.hadoop.util.StringUtils;
@@ -1528,41 +1526,6 @@ public class FSOperations {
}
/**
- * Executor that performs a getFileBlockLocations FileSystemAccess
- * file system operation.
- */
- @InterfaceAudience.Private
- @SuppressWarnings("rawtypes")
- public static class FSFileBlockLocations implements
- FileSystemAccess.FileSystemExecutor<Map> {
- private Path path;
- private long offsetValue;
- private long lengthValue;
-
- /**
- * Creates a file-block-locations executor.
- *
- * @param path the path to retrieve the location
- * @param offsetValue offset into the given file
- * @param lengthValue length for which to get locations for
- */
- public FSFileBlockLocations(String path, long offsetValue,
- long lengthValue) {
- this.path = new Path(path);
- this.offsetValue = offsetValue;
- this.lengthValue = lengthValue;
- }
-
- @Override
- public Map execute(FileSystem fs) throws IOException {
- BlockLocation[] locations =
- fs.getFileBlockLocations(this.path, this.offsetValue,
- this.lengthValue);
- return JsonUtil.toJsonMap(locations);
- }
- }
-
- /**
* Executor that performs a createSnapshot FileSystemAccess operation.
*/
@InterfaceAudience.Private
@@ -1665,5 +1628,4 @@ public class FSOperations {
return null;
}
}
-
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
index e8e75fe..384deca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
@@ -59,8 +59,7 @@ public class HttpFSParametersProvider extends ParametersProvider {
PARAMS_DEF.put(Operation.GETCONTENTSUMMARY, new Class[]{});
PARAMS_DEF.put(Operation.GETQUOTAUSAGE, new Class[]{});
PARAMS_DEF.put(Operation.GETFILECHECKSUM, new Class[]{});
- PARAMS_DEF.put(Operation.GETFILEBLOCKLOCATIONS,
- new Class[] {OffsetParam.class, LenParam.class});
+ PARAMS_DEF.put(Operation.GETFILEBLOCKLOCATIONS, new Class[]{});
PARAMS_DEF.put(Operation.GETACLSTATUS, new Class[]{});
PARAMS_DEF.put(Operation.GETTRASHROOT, new Class[]{});
PARAMS_DEF.put(Operation.INSTRUMENTATION, new Class[]{});
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
index 3d53cc5..23629c7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
@@ -51,7 +51,6 @@ import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.XAttrEncodingPa
import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.XAttrNameParam;
import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.XAttrSetFlagParam;
import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.XAttrValueParam;
-import org.apache.hadoop.hdfs.web.JsonUtil;
import org.apache.hadoop.lib.service.FileSystemAccess;
import org.apache.hadoop.lib.service.FileSystemAccessException;
import org.apache.hadoop.lib.service.Groups;
@@ -305,25 +304,7 @@ public class HttpFSServer {
break;
}
case GETFILEBLOCKLOCATIONS: {
- long offset = 0;
- // In case length is not given, reset to max long
- // in order to retrieve all file block locations
- long len = Long.MAX_VALUE;
- Long offsetParam = params.get(OffsetParam.NAME, OffsetParam.class);
- Long lenParam = params.get(LenParam.NAME, LenParam.class);
- AUDIT_LOG.info("[{}] offset [{}] len [{}]",
- new Object[] {path, offsetParam, lenParam});
- if (offsetParam != null && offsetParam.longValue() > 0) {
- offset = offsetParam.longValue();
- }
- if (lenParam != null && lenParam.longValue() > 0) {
- len = lenParam.longValue();
- }
- FSOperations.FSFileBlockLocations command =
- new FSOperations.FSFileBlockLocations(path, offset, len);
- @SuppressWarnings("rawtypes") Map locations = fsExecute(user, command);
- final String json = JsonUtil.toJsonString("BlockLocations", locations);
- response = Response.ok(json).type(MediaType.APPLICATION_JSON).build();
+ response = Response.status(Response.Status.BAD_REQUEST).build();
break;
}
case GETACLSTATUS: {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
index 9c9ece8..7d8943b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.fs.http.client;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockStoragePolicySpi;
-import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -1064,7 +1063,7 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
SET_REPLICATION, CHECKSUM, CONTENT_SUMMARY, QUOTA_USAGE, FILEACLS, DIRACLS,
SET_XATTR, GET_XATTRS, REMOVE_XATTR, LIST_XATTRS, ENCRYPTION,
LIST_STATUS_BATCH, GETTRASHROOT, STORAGEPOLICY, ERASURE_CODING,
- GETFILEBLOCKLOCATIONS, CREATE_SNAPSHOT, RENAME_SNAPSHOT, DELETE_SNAPSHOT
+ CREATE_SNAPSHOT, RENAME_SNAPSHOT, DELETE_SNAPSHOT
}
private void operation(Operation op) throws Exception {
@@ -1153,9 +1152,6 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
case STORAGEPOLICY:
testStoragePolicy();
break;
- case GETFILEBLOCKLOCATIONS:
- testGetFileBlockLocations();
- break;
case CREATE_SNAPSHOT:
testCreateSnapshot();
break;
@@ -1211,88 +1207,6 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
});
}
- private void testGetFileBlockLocations() throws Exception {
- BlockLocation[] locations1, locations2, locations11, locations21 = null;
- Path testFile = null;
-
- // Test single block file block locations.
- try (FileSystem fs = FileSystem.get(getProxiedFSConf())) {
- testFile = new Path(getProxiedFSTestDir(), "singleBlock.txt");
- DFSTestUtil.createFile(fs, testFile, (long) 1, (short) 1, 0L);
- locations1 = fs.getFileBlockLocations(testFile, 0, 1);
- Assert.assertNotNull(locations1);
- }
-
- try (FileSystem fs = getHttpFSFileSystem()) {
- locations2 = fs.getFileBlockLocations(testFile, 0, 1);
- Assert.assertNotNull(locations2);
- }
-
- verifyBlockLocations(locations1, locations2);
-
- // Test multi-block single replica file block locations.
- try (FileSystem fs = FileSystem.get(getProxiedFSConf())) {
- testFile = new Path(getProxiedFSTestDir(), "multipleBlocks.txt");
- DFSTestUtil.createFile(fs, testFile, 512, (short) 2048,
- (long) 512, (short) 1, 0L);
- locations1 = fs.getFileBlockLocations(testFile, 0, 1024);
- locations11 = fs.getFileBlockLocations(testFile, 1024, 2048);
- Assert.assertNotNull(locations1);
- Assert.assertNotNull(locations11);
- }
-
- try (FileSystem fs = getHttpFSFileSystem()) {
- locations2 = fs.getFileBlockLocations(testFile, 0, 1024);
- locations21 = fs.getFileBlockLocations(testFile, 1024, 2048);
- Assert.assertNotNull(locations2);
- Assert.assertNotNull(locations21);
- }
-
- verifyBlockLocations(locations1, locations2);
- verifyBlockLocations(locations11, locations21);
-
- // Test multi-block multi-replica file block locations.
- try (FileSystem fs = FileSystem.get(getProxiedFSConf())) {
- testFile = new Path(getProxiedFSTestDir(), "multipleBlocks.txt");
- DFSTestUtil.createFile(fs, testFile, 512, (short) 2048,
- (long) 512, (short) 3, 0L);
- locations1 = fs.getFileBlockLocations(testFile, 0, 2048);
- Assert.assertNotNull(locations1);
- }
-
- try (FileSystem fs = getHttpFSFileSystem()) {
- locations2 = fs.getFileBlockLocations(testFile, 0, 2048);
- Assert.assertNotNull(locations2);
- }
-
- verifyBlockLocations(locations1, locations2);
- }
-
- private void verifyBlockLocations(BlockLocation[] locations1,
- BlockLocation[] locations2) throws IOException {
- Assert.assertEquals(locations1.length, locations2.length);
- for (int i = 0; i < locations1.length; i++) {
- BlockLocation location1 = locations1[i];
- BlockLocation location2 = locations2[i];
-
- Assert.assertEquals(location1.isCorrupt(), location2.isCorrupt());
- Assert.assertEquals(location1.getOffset(), location2.getOffset());
- Assert.assertEquals(location1.getLength(), location2.getLength());
-
- Arrays.sort(location1.getHosts());
- Arrays.sort(location2.getHosts());
- Arrays.sort(location1.getNames());
- Arrays.sort(location2.getNames());
- Arrays.sort(location1.getTopologyPaths());
- Arrays.sort(location2.getTopologyPaths());
-
- Assert.assertArrayEquals(location1.getHosts(), location2.getHosts());
- Assert.assertArrayEquals(location1.getNames(), location2.getNames());
- Assert.assertArrayEquals(location1.getTopologyPaths(),
- location2.getTopologyPaths());
- }
- }
-
private void testCreateSnapshot(String snapshotName) throws Exception {
if (!this.isLocalFS()) {
Path snapshottablePath = new Path("/tmp/tmp-snap-test");
@@ -1385,5 +1299,4 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
fs.delete(snapshottablePath, true);
}
}
-
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org