You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-issues@hadoop.apache.org by GitBox <gi...@apache.org> on 2021/08/25 19:59:47 UTC
[GitHub] [hadoop] steveloughran commented on a change in pull request #2975: HADOOP-17682. ABFS: Support FileStatus input to OpenFileWithOptions() via OpenFileParameters
steveloughran commented on a change in pull request #2975:
URL: https://github.com/apache/hadoop/pull/2975#discussion_r696063619
##########
File path: hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
##########
@@ -669,44 +672,64 @@ public void createDirectory(final Path path, final FsPermission permission,
public AbfsInputStream openFileForRead(final Path path,
final FileSystem.Statistics statistics, TracingContext tracingContext)
- throws AzureBlobFileSystemException {
- return openFileForRead(path, Optional.empty(), statistics, tracingContext);
+ throws IOException {
+ return openFileForRead(path, Optional.empty(), statistics,
+ tracingContext);
}
- public AbfsInputStream openFileForRead(final Path path,
- final Optional<Configuration> options,
+ public AbfsInputStream openFileForRead(Path path,
+ final Optional<OpenFileParameters> parameters,
final FileSystem.Statistics statistics, TracingContext tracingContext)
- throws AzureBlobFileSystemException {
- try (AbfsPerfInfo perfInfo = startTracking("openFileForRead", "getPathStatus")) {
+ throws IOException {
+ try (AbfsPerfInfo perfInfo = startTracking("openFileForRead",
+ "getPathStatus")) {
LOG.debug("openFileForRead filesystem: {} path: {}",
- client.getFileSystem(),
- path);
+ client.getFileSystem(), path);
+ FileStatus fileStatus = parameters.map(OpenFileParameters::getStatus)
+ .orElse(null);
String relativePath = getRelativePath(path);
-
- final AbfsRestOperation op = client
- .getPathStatus(relativePath, false, tracingContext);
- perfInfo.registerResult(op.getResult());
-
- final String resourceType = op.getResult().getResponseHeader(HttpHeaderConfigurations.X_MS_RESOURCE_TYPE);
- final long contentLength = Long.parseLong(op.getResult().getResponseHeader(HttpHeaderConfigurations.CONTENT_LENGTH));
- final String eTag = op.getResult().getResponseHeader(HttpHeaderConfigurations.ETAG);
+ String resourceType, eTag;
+ long contentLength;
+ if (fileStatus instanceof VersionedFileStatus) {
+ path = path.makeQualified(this.uri, path);
+ Preconditions.checkArgument(fileStatus.getPath().equals(path),
+ String.format(
+ "Filestatus path [%s] does not match with given path [%s]",
+ fileStatus.getPath(), path));
+ resourceType = fileStatus.isFile() ? FILE : DIRECTORY;
+ contentLength = fileStatus.getLen();
+ eTag = ((VersionedFileStatus) fileStatus).getVersion();
+ } else {
+ if (fileStatus != null) {
+ LOG.warn(
Review comment:
no, just use length & type, don't worry about etag. Or at least just log @ debug
##########
File path: hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java
##########
@@ -192,6 +201,106 @@ public TestAbfsInputStream() throws Exception {
ReadBufferManager.getBufferManager().setThresholdAgeMilliseconds(REDUCED_READ_BUFFER_AGE_THRESHOLD);
}
+ private void writeBufferToNewFile(Path testFile, byte[] buffer) throws IOException {
+ AzureBlobFileSystem fs = getFileSystem();
+ fs.create(testFile);
+ FSDataOutputStream out = fs.append(testFile);
+ out.write(buffer);
+ out.close();
+ }
+
+ private void verifyOpenWithProvidedStatus(Path path, FileStatus fileStatus,
+ byte[] buf, AbfsRestOperationType source)
+ throws IOException, ExecutionException, InterruptedException {
+ byte[] readBuf = new byte[buf.length];
+ AzureBlobFileSystem fs = getFileSystem();
+ FutureDataInputStreamBuilder builder = fs.openFile(path);
+ builder.withFileStatus(fileStatus);
+ FSDataInputStream in = builder.build().get();
+ assertEquals(String.format(
Review comment:
use AssertJ
##########
File path: hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
##########
@@ -669,44 +672,64 @@ public void createDirectory(final Path path, final FsPermission permission,
public AbfsInputStream openFileForRead(final Path path,
final FileSystem.Statistics statistics, TracingContext tracingContext)
- throws AzureBlobFileSystemException {
- return openFileForRead(path, Optional.empty(), statistics, tracingContext);
+ throws IOException {
+ return openFileForRead(path, Optional.empty(), statistics,
+ tracingContext);
}
- public AbfsInputStream openFileForRead(final Path path,
- final Optional<Configuration> options,
+ public AbfsInputStream openFileForRead(Path path,
+ final Optional<OpenFileParameters> parameters,
final FileSystem.Statistics statistics, TracingContext tracingContext)
- throws AzureBlobFileSystemException {
- try (AbfsPerfInfo perfInfo = startTracking("openFileForRead", "getPathStatus")) {
+ throws IOException {
+ try (AbfsPerfInfo perfInfo = startTracking("openFileForRead",
+ "getPathStatus")) {
LOG.debug("openFileForRead filesystem: {} path: {}",
- client.getFileSystem(),
- path);
+ client.getFileSystem(), path);
+ FileStatus fileStatus = parameters.map(OpenFileParameters::getStatus)
+ .orElse(null);
String relativePath = getRelativePath(path);
-
- final AbfsRestOperation op = client
- .getPathStatus(relativePath, false, tracingContext);
- perfInfo.registerResult(op.getResult());
-
- final String resourceType = op.getResult().getResponseHeader(HttpHeaderConfigurations.X_MS_RESOURCE_TYPE);
- final long contentLength = Long.parseLong(op.getResult().getResponseHeader(HttpHeaderConfigurations.CONTENT_LENGTH));
- final String eTag = op.getResult().getResponseHeader(HttpHeaderConfigurations.ETAG);
+ String resourceType, eTag;
+ long contentLength;
+ if (fileStatus instanceof VersionedFileStatus) {
+ path = path.makeQualified(this.uri, path);
+ Preconditions.checkArgument(fileStatus.getPath().equals(path),
+ String.format(
+ "Filestatus path [%s] does not match with given path [%s]",
+ fileStatus.getPath(), path));
+ resourceType = fileStatus.isFile() ? FILE : DIRECTORY;
+ contentLength = fileStatus.getLen();
+ eTag = ((VersionedFileStatus) fileStatus).getVersion();
+ } else {
+ if (fileStatus != null) {
+ LOG.warn(
+ "Fallback to getPathStatus REST call as provided filestatus "
+ + "is not of type VersionedFileStatus");
+ }
+ AbfsHttpOperation op = client.getPathStatus(relativePath, false,
+ tracingContext).getResult();
+ resourceType = op.getResponseHeader(
+ HttpHeaderConfigurations.X_MS_RESOURCE_TYPE);
+ contentLength = Long.parseLong(
+ op.getResponseHeader(HttpHeaderConfigurations.CONTENT_LENGTH));
+ eTag = op.getResponseHeader(HttpHeaderConfigurations.ETAG);
+ }
if (parseIsDirectory(resourceType)) {
throw new AbfsRestOperationException(
Review comment:
is there a test for this
##########
File path: hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
##########
@@ -669,44 +672,64 @@ public void createDirectory(final Path path, final FsPermission permission,
public AbfsInputStream openFileForRead(final Path path,
final FileSystem.Statistics statistics, TracingContext tracingContext)
- throws AzureBlobFileSystemException {
- return openFileForRead(path, Optional.empty(), statistics, tracingContext);
+ throws IOException {
+ return openFileForRead(path, Optional.empty(), statistics,
+ tracingContext);
}
- public AbfsInputStream openFileForRead(final Path path,
- final Optional<Configuration> options,
+ public AbfsInputStream openFileForRead(Path path,
+ final Optional<OpenFileParameters> parameters,
final FileSystem.Statistics statistics, TracingContext tracingContext)
- throws AzureBlobFileSystemException {
- try (AbfsPerfInfo perfInfo = startTracking("openFileForRead", "getPathStatus")) {
+ throws IOException {
+ try (AbfsPerfInfo perfInfo = startTracking("openFileForRead",
+ "getPathStatus")) {
LOG.debug("openFileForRead filesystem: {} path: {}",
- client.getFileSystem(),
- path);
+ client.getFileSystem(), path);
+ FileStatus fileStatus = parameters.map(OpenFileParameters::getStatus)
+ .orElse(null);
String relativePath = getRelativePath(path);
-
- final AbfsRestOperation op = client
- .getPathStatus(relativePath, false, tracingContext);
- perfInfo.registerResult(op.getResult());
-
- final String resourceType = op.getResult().getResponseHeader(HttpHeaderConfigurations.X_MS_RESOURCE_TYPE);
- final long contentLength = Long.parseLong(op.getResult().getResponseHeader(HttpHeaderConfigurations.CONTENT_LENGTH));
- final String eTag = op.getResult().getResponseHeader(HttpHeaderConfigurations.ETAG);
+ String resourceType, eTag;
+ long contentLength;
+ if (fileStatus instanceof VersionedFileStatus) {
+ path = path.makeQualified(this.uri, path);
+ Preconditions.checkArgument(fileStatus.getPath().equals(path),
Review comment:
only check getPath().getName() for equivalence.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org