You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by bh...@apache.org on 2021/11/17 05:58:43 UTC
[ozone] branch master updated: HDDS-5964. [S3-TDE] Range Reads not working on encrypted buckets (#2827)
This is an automated email from the ASF dual-hosted git repository.
bharat pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 6ea9916 HDDS-5964. [S3-TDE] Range Reads not working on encrypted buckets (#2827)
6ea9916 is described below
commit 6ea9916260636354b84426e6e76952085ea3b063
Author: Bharat Viswanadham <bh...@apache.org>
AuthorDate: Tue Nov 16 21:58:18 2021 -0800
HDDS-5964. [S3-TDE] Range Reads not working on encrypted buckets (#2827)
---
.../hadoop/ozone/client/io/OzoneInputStream.java | 33 ++++++++-
.../src/main/compose/ozonesecure-mr/docker-config | 3 +
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 10 +--
.../hadoop/ozone/s3/io/S3WrapperInputStream.java | 84 ----------------------
.../apache/hadoop/ozone/s3/io/package-info.java | 23 ------
5 files changed, 38 insertions(+), 115 deletions(-)
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneInputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneInputStream.java
index 548a20e..937a9b7 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneInputStream.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneInputStream.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.client.io;
import org.apache.hadoop.fs.ByteBufferReadable;
import org.apache.hadoop.fs.CanUnbuffer;
+import org.apache.hadoop.fs.Seekable;
import java.io.IOException;
import java.io.InputStream;
@@ -29,7 +30,7 @@ import java.nio.ByteBuffer;
* It uses {@link KeyInputStream} for reading the data.
*/
public class OzoneInputStream extends InputStream implements CanUnbuffer,
- ByteBufferReadable {
+ ByteBufferReadable, Seekable {
private final InputStream inputStream;
@@ -90,4 +91,34 @@ public class OzoneInputStream extends InputStream implements CanUnbuffer,
((CanUnbuffer) inputStream).unbuffer();
}
}
+
+ @Override
+ public void seek(long pos) throws IOException {
+ if (inputStream instanceof Seekable) {
+ ((Seekable) inputStream).seek(pos);
+ } else {
+ throw new UnsupportedOperationException("Seek is not supported on the " +
+ "underlying inputStream");
+ }
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ if (inputStream instanceof Seekable) {
+ return ((Seekable) inputStream).getPos();
+ } else {
+ throw new UnsupportedOperationException("Seek is not supported on the " +
+ "underlying inputStream");
+ }
+ }
+
+ @Override
+ public boolean seekToNewSource(long targetPos) throws IOException {
+ if (inputStream instanceof Seekable) {
+ return ((Seekable) inputStream).seekToNewSource(targetPos);
+ } else {
+ throw new UnsupportedOperationException("Seek is not supported on the " +
+ "underlying inputStream");
+ }
+ }
}
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
index cdf3f67..942b45b 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
@@ -47,6 +47,9 @@ OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab=/etc/security/keytabs/s3g.key
OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal=HTTP/s3g@EXAMPLE.COM
OZONE-SITE.XML_hdds.grpc.tls.enabled=true
+OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file=/etc/security/keytabs/s3g.keytab
+OZONE-SITE.XML_ozone.s3g.kerberos.principal=s3g/s3g@EXAMPLE.COM
+
HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/dn@EXAMPLE.COM
HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/dn@EXAMPLE.COM
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index 64af127..ad39658 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -72,7 +72,6 @@ import org.apache.hadoop.ozone.s3.HeaderPreprocessor;
import org.apache.hadoop.ozone.s3.SignedChunksInputStream;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
-import org.apache.hadoop.ozone.s3.io.S3WrapperInputStream;
import org.apache.hadoop.ozone.s3.util.RFC1123Util;
import org.apache.hadoop.ozone.s3.util.RangeHeader;
import org.apache.hadoop.ozone.s3.util.RangeHeaderParserUtil;
@@ -291,7 +290,6 @@ public class ObjectEndpoint extends EndpointBase {
.header(CONTENT_LENGTH, keyDetails.getDataSize());
} else {
- OzoneInputStream key = bucket.readKey(keyPath);
long startOffset = rangeHeader.getStartOffset();
long endOffset = rangeHeader.getEndOffset();
@@ -299,11 +297,9 @@ public class ObjectEndpoint extends EndpointBase {
// byte from start offset
long copyLength = endOffset - startOffset + 1;
StreamingOutput output = dest -> {
- try (S3WrapperInputStream s3WrapperInputStream =
- new S3WrapperInputStream(
- key.getInputStream())) {
- s3WrapperInputStream.seek(startOffset);
- IOUtils.copyLarge(s3WrapperInputStream, dest, 0,
+ try (OzoneInputStream ozoneInputStream = bucket.readKey(keyPath)) {
+ ozoneInputStream.seek(startOffset);
+ IOUtils.copyLarge(ozoneInputStream, dest, 0,
copyLength, new byte[bufferSize]);
}
};
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/S3WrapperInputStream.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/S3WrapperInputStream.java
deleted file mode 100644
index d88287c..0000000
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/S3WrapperInputStream.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.s3.io;
-
-import org.apache.hadoop.fs.FSInputStream;
-import org.apache.hadoop.ozone.client.io.KeyInputStream;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * S3Wrapper Input Stream which encapsulates KeyInputStream from ozone.
- */
-public class S3WrapperInputStream extends FSInputStream {
- private final KeyInputStream inputStream;
-
- /**
- * Constructs S3WrapperInputStream with KeyInputStream.
- *
- * @param inputStream
- */
- public S3WrapperInputStream(InputStream inputStream) {
- this.inputStream = (KeyInputStream) inputStream;
- }
-
- @Override
- public int read() throws IOException {
- return inputStream.read();
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- return inputStream.read(b, off, len);
- }
-
- @Override
- public synchronized void close() throws IOException {
- inputStream.close();
- }
-
- @Override
- public int available() throws IOException {
- return inputStream.available();
- }
-
- public InputStream getInputStream() {
- return inputStream;
- }
-
- @Override
- public void seek(long pos) throws IOException {
- inputStream.seek(pos);
- }
- @Override
- public long getPos() throws IOException {
- return inputStream.getPos();
- }
-
- @Override
- public long skip(long n) throws IOException {
- return inputStream.skip(n);
- }
-
- @Override
- public boolean seekToNewSource(long targetPos) throws IOException {
- return false;
- }
-}
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/package-info.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/package-info.java
deleted file mode 100644
index 5167e60..0000000
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * This package contains Ozone S3 wrapper stream related classes.
- */
-
-package org.apache.hadoop.ozone.s3.io;
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org